{"id":"https://openalex.org/W3206530948","doi":"https://doi.org/10.1145/3479586","title":"Quality Control in Crowdsourcing based on Fine-Grained Behavioral Features","display_name":"Quality Control in Crowdsourcing based on Fine-Grained Behavioral Features","publication_year":2021,"publication_date":"2021-10-13","ids":{"openalex":"https://openalex.org/W3206530948","doi":"https://doi.org/10.1145/3479586","mag":"3206530948"},"language":"en","primary_location":{"id":"doi:10.1145/3479586","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3479586","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3479586","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3479586","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089019010","display_name":"Weiping Pei","orcid":"https://orcid.org/0000-0002-6831-2313"},"institutions":[{"id":"https://openalex.org/I167576493","display_name":"Colorado School of Mines","ror":"https://ror.org/04raf6v53","country_code":"US","type":"education","lineage":["https://openalex.org/I167576493"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Weiping Pei","raw_affiliation_strings":["Colorado School of Mines, Golden, CO, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Colorado School of Mines, Golden, CO, USA","institution_ids":["https://openalex.org/I167576493"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086262095","display_name":"Zhiju Yang","orcid":null},"institutions":[{"id":"https://openalex.org/I167576493","display_name":"Colorado School of Mines","ror":"https://ror.org/04raf6v53","country_code":"US","type":"education","lineage":["https://openalex.org/I167576493"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhiju Yang","raw_affiliation_strings":["Colorado School of Mines, Golden, CO, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Colorado School of Mines, Golden, CO, USA","institution_ids":["https://openalex.org/I167576493"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076454817","display_name":"Monchu Chen","orcid":"https://orcid.org/0000-0001-7242-8442"},"institutions":[{"id":"https://openalex.org/I4210123540","display_name":"Appen (United States)","ror":"https://ror.org/029pncr02","country_code":"US","type":"company","lineage":["https://openalex.org/I4210123540"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Monchu Chen","raw_affiliation_strings":["Appen, Sunnyvale, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Appen, Sunnyvale, CA, USA","institution_ids":["https://openalex.org/I4210123540"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101641878","display_name":"Chuan Yue","orcid":"https://orcid.org/0000-0002-6095-4768"},"institutions":[{"id":"https://openalex.org/I167576493","display_name":"Colorado School of Mines","ror":"https://ror.org/04raf6v53","country_code":"US","type":"education","lineage":["https://openalex.org/I167576493"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chuan Yue","raw_affiliation_strings":["Colorado School of Mines, Golden, CO, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Colorado School of Mines, Golden, CO, USA","institution_ids":["https://openalex.org/I167576493"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.1608,"has_fulltext":true,"cited_by_count":8,"citation_normalized_percentile":{"value":0.82706354,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":"5","issue":"CSCW2","first_page":"1","last_page":"28"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9758999943733215,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9682000279426575,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9836140275001526},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.7371184229850769},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7237863540649414},{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.7028269171714783},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.6202049851417542},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4545239508152008},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.39274391531944275},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3393383026123047},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.13276106119155884}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9836140275001526},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.7371184229850769},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7237863540649414},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.7028269171714783},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.6202049851417542},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4545239508152008},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39274391531944275},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3393383026123047},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.13276106119155884},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3479586","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3479586","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3479586","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3479586","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3479586","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3479586","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/8","score":0.5600000023841858,"display_name":"Decent work and economic growth"}],"awards":[{"id":"https://openalex.org/G2731861321","display_name":"Convergence Accelerator Phase I (RAISE): Toward Fair, Ethical, Efficient, and Trustworthy Crowdsourcing Platforms to Support Crowdworkers in Jobs of the Future","funder_award_id":"1936968","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4957512964","display_name":null,"funder_award_id":"OIA-1936968","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3206530948.pdf","grobid_xml":"https://content.openalex.org/works/W3206530948.grobid-xml"},"referenced_works_count":36,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W1532325895","https://openalex.org/W1565732937","https://openalex.org/W1570705485","https://openalex.org/W2030139727","https://openalex.org/W2036431079","https://openalex.org/W2064675550","https://openalex.org/W2093044076","https://openalex.org/W2114100913","https://openalex.org/W2116705992","https://openalex.org/W2124994029","https://openalex.org/W2150597317","https://openalex.org/W2292974053","https://openalex.org/W2296739333","https://openalex.org/W2315338392","https://openalex.org/W2343442396","https://openalex.org/W2529665187","https://openalex.org/W2535526893","https://openalex.org/W2558487799","https://openalex.org/W2579968393","https://openalex.org/W2777647957","https://openalex.org/W2811107472","https://openalex.org/W2908879514","https://openalex.org/W2911656801","https://openalex.org/W2950628590","https://openalex.org/W2952395386","https://openalex.org/W2962766617","https://openalex.org/W2963686995","https://openalex.org/W3010595911","https://openalex.org/W3012845547","https://openalex.org/W3093582825","https://openalex.org/W3094043653","https://openalex.org/W3100373746","https://openalex.org/W3121480429","https://openalex.org/W4213009331","https://openalex.org/W4288083516"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3207526114","https://openalex.org/W4286908577"],"abstract_inverted_index":{"Crowdsourcing":[0],"is":[1,13,162],"popular":[2],"for":[3,98,130,137,178],"large-scale":[4],"data":[5,31],"collection":[6],"and":[7,33,49,72,105,140,155,164,176],"labeling,":[8],"but":[9],"a":[10,87,96,107],"major":[11],"challenge":[12],"on":[14],"detecting":[15],"low-quality":[16],"submissions.":[17],"Recent":[18],"studies":[19,42],"have":[20],"demonstrated":[21],"that":[22,115,157],"behavioral":[23,47,77,82,119,160],"features":[24,83,120,161],"of":[25,74],"workers":[26],"are":[27,80],"highly":[28],"correlated":[29],"with":[30,91],"quality":[32,38,54,99,124,128,169,189],"can":[34],"be":[35],"useful":[36],"in":[37,95,101,166],"control.":[39,190],"However,":[40],"these":[41],"primarily":[43],"leveraged":[44],"coarsely":[45],"extracted":[46,85],"features,":[48,78],"did":[50],"not":[51],"further":[52,186],"explore":[53],"control":[55,100,125,170],"at":[56],"the":[57,61,70,81,146],"fine-grained":[58,76,118,159],"level,":[59],"i.e.,":[60],"annotation":[62],"unit":[63,94],"level.":[64],"In":[65],"this":[66],"paper,":[67],"we":[68,149],"investigate":[69],"feasibility":[71],"benefits":[73],"using":[75,158],"which":[79],"finely":[84],"from":[86],"worker's":[88],"individual":[89],"interactions":[90],"each":[92],"single":[93],"subtask,":[97],"crowdsourcing.":[102],"We":[103],"design":[104],"implement":[106],"framework":[108],"named":[109],"Fine-grained":[110],"Behavior-based":[111],"Quality":[112],"Control":[113],"(FBQC)":[114],"specifically":[116],"extracts":[117],"to":[121,185],"provide":[122],"three":[123,168],"mechanisms:":[126],"(1)":[127],"prediction":[129],"objective":[131],"tasks,":[132,139],"(2)":[133],"suspicious":[134],"behavior":[135],"detection":[136],"subjective":[138],"(3)":[141],"unsupervised":[142],"worker":[143],"categorization.":[144],"Using":[145],"FBQC":[147],"framework,":[148],"conduct":[150],"two":[151],"real-world":[152],"crowdsourcing":[153,183],"experiments":[154],"demonstrate":[156],"feasible":[163],"beneficial":[165],"all":[167],"mechanisms.":[171],"Our":[172],"work":[173],"provides":[174],"clues":[175],"implications":[177],"helping":[179],"job":[180],"requesters":[181],"or":[182],"platforms":[184],"achieve":[187],"better":[188]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":4}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}