{"id":"https://openalex.org/W2292021238","doi":"https://doi.org/10.1109/iwqos.2015.7404734","title":"Detecting low-quality crowdtesting workers","display_name":"Detecting low-quality crowdtesting workers","publication_year":2015,"publication_date":"2015-06-01","ids":{"openalex":"https://openalex.org/W2292021238","doi":"https://doi.org/10.1109/iwqos.2015.7404734","mag":"2292021238"},"language":"en","primary_location":{"id":"doi:10.1109/iwqos.2015.7404734","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iwqos.2015.7404734","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2015 IEEE 23rd International Symposium on Quality of Service (IWQoS)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5084986707","display_name":"Ricky K. P. Mok","orcid":"https://orcid.org/0000-0003-3300-9514"},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Ricky K. P. Mok","raw_affiliation_strings":["Department of Computing The Hong Kong Polytechnic University","Department of Computing, The Hong Kong Polytechnic University#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computing The Hong Kong Polytechnic University","institution_ids":["https://openalex.org/I14243506"]},{"raw_affiliation_string":"Department of Computing, The Hong Kong Polytechnic University#TAB#","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100637381","display_name":"Weichao Li","orcid":"https://orcid.org/0000-0002-7620-1955"},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Weichao Li","raw_affiliation_strings":["Department of Computing The Hong Kong Polytechnic University","Department of Computing, The Hong Kong Polytechnic University#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computing The Hong Kong Polytechnic University","institution_ids":["https://openalex.org/I14243506"]},{"raw_affiliation_string":"Department of Computing, The Hong Kong Polytechnic University#TAB#","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5006486786","display_name":"Rocky K. C. Chang","orcid":"https://orcid.org/0000-0002-2648-5814"},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Rocky K. C. Chang","raw_affiliation_strings":["Department of Computing The Hong Kong Polytechnic University","Department of Computing, The Hong Kong Polytechnic University#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computing The Hong Kong Polytechnic University","institution_ids":["https://openalex.org/I14243506"]},{"raw_affiliation_string":"Department of Computing, The Hong Kong Polytechnic University#TAB#","institution_ids":["https://openalex.org/I14243506"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I14243506"],"apc_list":null,"apc_paid":null,"fwci":1.434,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.83201957,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":"16","issue":null,"first_page":"201","last_page":"206"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9882000088691711,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11500","display_name":"Evacuation and Crowd Dynamics","score":0.9787999987602234,"subfield":{"id":"https://openalex.org/subfields/2212","display_name":"Ocean Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7936479449272156},{"id":"https://openalex.org/keywords/cheating","display_name":"Cheating","score":0.6994813680648804},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.6937641501426697},{"id":"https://openalex.org/keywords/likert-scale","display_name":"Likert scale","score":0.5829309225082397},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.506013810634613},{"id":"https://openalex.org/keywords/naive-bayes-classifier","display_name":"Naive Bayes classifier","score":0.5043374300003052},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4810248017311096},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.47738856077194214},{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.4612966477870941},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.45168715715408325},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4264459013938904},{"id":"https://openalex.org/keywords/cursor","display_name":"Cursor (databases)","score":0.4260011613368988},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.36287128925323486},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.14490088820457458},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.10598331689834595},{"id":"https://openalex.org/keywords/support-vector-machine","display_name":"Support vector machine","score":0.10359212756156921},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09549763798713684}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7936479449272156},{"id":"https://openalex.org/C2778024590","wikidata":"https://www.wikidata.org/wiki/Q2357432","display_name":"Cheating","level":2,"score":0.6994813680648804},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.6937641501426697},{"id":"https://openalex.org/C105776082","wikidata":"https://www.wikidata.org/wiki/Q617473","display_name":"Likert scale","level":2,"score":0.5829309225082397},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.506013810634613},{"id":"https://openalex.org/C52001869","wikidata":"https://www.wikidata.org/wiki/Q812530","display_name":"Naive Bayes classifier","level":3,"score":0.5043374300003052},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4810248017311096},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.47738856077194214},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.4612966477870941},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.45168715715408325},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4264459013938904},{"id":"https://openalex.org/C2776990265","wikidata":"https://www.wikidata.org/wiki/Q2998101","display_name":"Cursor (databases)","level":2,"score":0.4260011613368988},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.36287128925323486},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.14490088820457458},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.10598331689834595},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.10359212756156921},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09549763798713684},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C1276947","wikidata":"https://www.wikidata.org/wiki/Q333","display_name":"Astronomy","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/iwqos.2015.7404734","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iwqos.2015.7404734","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2015 IEEE 23rd International Symposium on Quality of Service (IWQoS)","raw_type":"proceedings-article"},{"id":"pmh:oai:ira.lib.polyu.edu.hk:10397/73707","is_oa":false,"landing_page_url":"http://hdl.handle.net/10397/73707","pdf_url":null,"source":{"id":"https://openalex.org/S4306400205","display_name":"PolyU Institutional Research Archive (Hong Kong Polytechnic University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I14243506","host_organization_name":"Hong Kong Polytechnic University","host_organization_lineage":["https://openalex.org/I14243506"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Paper"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.6700000166893005}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W156959604","https://openalex.org/W1909003218","https://openalex.org/W1969235627","https://openalex.org/W1986638496","https://openalex.org/W2026538633","https://openalex.org/W2044127330","https://openalex.org/W2044885272","https://openalex.org/W2048498434","https://openalex.org/W2061844755","https://openalex.org/W2101086718","https://openalex.org/W2104644670","https://openalex.org/W2104749423","https://openalex.org/W2120909761","https://openalex.org/W2124994029","https://openalex.org/W2135101375","https://openalex.org/W2135626977","https://openalex.org/W2140336868","https://openalex.org/W2158495800","https://openalex.org/W2160473997","https://openalex.org/W2163031058","https://openalex.org/W2167913131","https://openalex.org/W2292984643","https://openalex.org/W2315338392","https://openalex.org/W4244811101","https://openalex.org/W6606258704","https://openalex.org/W6639818920","https://openalex.org/W6681046083","https://openalex.org/W6697163159"],"related_works":["https://openalex.org/W2012288173","https://openalex.org/W3032998312","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W1503094549","https://openalex.org/W1968538666","https://openalex.org/W2097662580","https://openalex.org/W3199302685","https://openalex.org/W2344072770","https://openalex.org/W2389163612"],"abstract_inverted_index":{"QoE":[0],"crowdtesting":[1,35],"is":[2,36,58,68,115,207,233],"increasingly":[3],"popular":[4],"among":[5],"researchers":[6],"to":[7,18,37,69,99,108,116,125,137,165,178],"conduct":[8],"subjective":[9],"assessments":[10],"of":[11,22,34,49,56,120,129,154,186,200],"different":[12,127,144,180],"services.":[13],"Experimenters":[14],"can":[15],"easily":[16],"access":[17],"a":[19,118],"huge":[20],"pool":[21],"human":[23,161],"subjects":[24],"through":[25],"crowdsourcing":[26],"platforms.":[27],"A":[28,65,132,152],"fundamental":[29],"problem":[30],"threatening":[31],"the":[32,41,50,54,63,72,81,88,92,139,187,190,197,201,219,222,226],"integrity":[33],"detect":[38],"cheating":[39],"from":[40,204,221],"workers":[42,57,167,232],"who":[43],"work":[44],"without":[45],"any":[46],"supervision.":[47],"One":[48],"approaches":[51],"in":[52,91,229],"classifying":[53],"quality":[55,111],"analyzing":[59],"their":[60],"behavior":[61,123],"during":[62],"experiments.":[64],"major":[66],"challenge":[67],"systematically":[70],"analyze":[71,80],"mouse":[73],"cursor":[74,102],"trajectory.":[75,93],"However,":[76],"existing":[77],"works":[78],"usually":[79],"trajectory":[82,103],"coarsely,":[83],"which":[84],"cannot":[85],"fully":[86],"extract":[87],"information":[89],"imbedded":[90],"In":[94],"this":[95],"paper,":[96],"we":[97],"propose":[98],"use":[100],"finer-grained":[101],"analysis,":[104,107],"including":[105,156],"submovement":[106],"identify":[109],"low":[110],"workers.":[112],"Our":[113,193],"approach":[114],"define":[117],"set":[119],"ten":[121],"worker":[122,130,140],"metrics":[124,188,206],"quantify":[126],"types":[128],"behavior.":[131,141],"jQuery-based":[133],"library":[134],"was":[135],"implemented":[136],"collect":[138],"Moreover,":[142],"four":[143,205,214,223],"5-point":[145],"Likert":[146],"scale":[147],"rating":[148,215,224],"methods":[149],"were":[150],"employed.":[151],"number":[153],"methods,":[155,225],"question":[157],"design,":[158],"instructions,":[159],"and":[160,189],"inspections,":[162],"are":[163],"used":[164],"label":[166],"into":[168],"three":[169],"categories.":[170],"We":[171],"then":[172],"apply":[173],"multiclass":[174],"Naive":[175],"Bayes":[176],"classifier":[177],"construct":[179],"models":[181],"using":[182],"all":[183],"or":[184,209],"some":[185],"workers'":[191],"category.":[192],"results":[194],"show":[195],"that":[196],"error":[198],"rates":[199],"model":[202],"trained":[203],"equal":[208],"less":[210],"than":[211],"30%":[212],"for":[213],"methods.":[216],"By":[217],"combining":[218],"predictions":[220],"successful":[227],"rate":[228],"detecting":[230],"low-quality":[231],"around":[234],"80%.":[235]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":2}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}