{"id":"https://openalex.org/W4296079293","doi":"https://doi.org/10.1109/access.2022.3207292","title":"What Quality Control Mechanisms do We Need for High-Quality Crowd Work?","display_name":"What Quality Control Mechanisms do We Need for High-Quality Crowd Work?","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4296079293","doi":"https://doi.org/10.1109/access.2022.3207292"},"language":"en","primary_location":{"id":"doi:10.1109/access.2022.3207292","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2022.3207292","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/09893786.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/09893786.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050131784","display_name":"Margeret Hall","orcid":"https://orcid.org/0000-0003-1049-3040"},"institutions":[{"id":"https://openalex.org/I102248843","display_name":"Vienna University of Economics and Business","ror":"https://ror.org/03yn8s215","country_code":"AT","type":"education","lineage":["https://openalex.org/I102248843"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Margeret Hall","raw_affiliation_strings":["Department of Strategy and Innovation, Wirtschaftsuniversität Wien, Vienna, Austria"],"raw_orcid":"https://orcid.org/0000-0003-1049-3040","affiliations":[{"raw_affiliation_string":"Department of Strategy and Innovation, Wirtschaftsuniversität Wien, Vienna, Austria","institution_ids":["https://openalex.org/I102248843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027952619","display_name":"Farhad Mohammad Afzali","orcid":"https://orcid.org/0000-0001-7457-9148"},"institutions":[{"id":"https://openalex.org/I122266389","display_name":"University of Nebraska at Omaha","ror":"https://ror.org/04yrkc140","country_code":"US","type":"education","lineage":["https://openalex.org/I122266389"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mohammad Farhad Afzali","raw_affiliation_strings":["College of Information Science and Technology, University of Nebraska Omaha, Omaha, NE, USA"],"raw_orcid":"https://orcid.org/0000-0001-7457-9148","affiliations":[{"raw_affiliation_string":"College of Information Science and Technology, University of Nebraska Omaha, Omaha, NE, USA","institution_ids":["https://openalex.org/I122266389"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110564894","display_name":"Markus Krause","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Markus Krause","raw_affiliation_strings":["Brainworks.ai, Berkeley, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brainworks.ai, Berkeley, CA, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021197359","display_name":"Simon Caton","orcid":"https://orcid.org/0000-0001-9379-3879"},"institutions":[{"id":"https://openalex.org/I100930933","display_name":"University College Dublin","ror":"https://ror.org/05m7pjf47","country_code":"IE","type":"education","lineage":["https://openalex.org/I100930933"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Simon Caton","raw_affiliation_strings":["School of Computer Science, University College Dublin, Dublin, Ireland"],"raw_orcid":"https://orcid.org/0000-0001-9379-3879","affiliations":[{"raw_affiliation_string":"School of Computer Science, University College Dublin, Dublin, Ireland","institution_ids":["https://openalex.org/I100930933"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":0.8168,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.79434775,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"10","issue":null,"first_page":"99709","last_page":"99723"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.983299970626831,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11500","display_name":"Evacuation and Crowd Dynamics","score":0.972100019454956,"subfield":{"id":"https://openalex.org/subfields/2212","display_name":"Ocean Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6519678831100464},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6376281380653381},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.5214444994926453},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.5134323835372925},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.17393019795417786},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10071170330047607}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6519678831100464},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6376281380653381},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.5214444994926453},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.5134323835372925},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.17393019795417786},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10071170330047607},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/access.2022.3207292","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2022.3207292","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/09893786.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:research.wu.ac.at:publications/57eae93f-9138-41a9-a7f6-6b66f20bb85f","is_oa":false,"landing_page_url":"https://research.wu.ac.at/de/publications/57eae93f-9138-41a9-a7f6-6b66f20bb85f","pdf_url":null,"source":{"id":"https://openalex.org/S7407055123","display_name":"WU Research","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Hall, M, Afzali, M F, Krause, M & Caton, S 2022, 'What quality control mechanisms do we need for high-quality crowd work?', IEEE Access, vol. 10, pp. 99709-99723. https://doi.org/10.1109/ACCESS.2022.3207292","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:doaj.org/article:c888ebfe45ae45cd8fee2db92bd19ed7","is_oa":true,"landing_page_url":"https://doaj.org/article/c888ebfe45ae45cd8fee2db92bd19ed7","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 10, Pp 99709-99723 (2022)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2022.3207292","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2022.3207292","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/09893786.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.550000011920929}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4296079293.pdf","grobid_xml":"https://content.openalex.org/works/W4296079293.grobid-xml"},"referenced_works_count":94,"referenced_works":["https://openalex.org/W126134487","https://openalex.org/W167167890","https://openalex.org/W1558108873","https://openalex.org/W1677182931","https://openalex.org/W1820962962","https://openalex.org/W1831729862","https://openalex.org/W1912668879","https://openalex.org/W1941810463","https://openalex.org/W1972253352","https://openalex.org/W1974474704","https://openalex.org/W2013378101","https://openalex.org/W2019029324","https://openalex.org/W2026487812","https://openalex.org/W2035874553","https://openalex.org/W2036431079","https://openalex.org/W2039241549","https://openalex.org/W2041496158","https://openalex.org/W2049633694","https://openalex.org/W2058738785","https://openalex.org/W2064856757","https://openalex.org/W2067438047","https://openalex.org/W2070248097","https://openalex.org/W2078006360","https://openalex.org/W2080104152","https://openalex.org/W2085880784","https://openalex.org/W2091759811","https://openalex.org/W2092719662","https://openalex.org/W2098865355","https://openalex.org/W2101234009","https://openalex.org/W2103519186","https://openalex.org/W2106445101","https://openalex.org/W2108598243","https://openalex.org/W2119298903","https://openalex.org/W2124994029","https://openalex.org/W2125943921","https://openalex.org/W2135504089","https://openalex.org/W2151401338","https://openalex.org/W2160505695","https://openalex.org/W2170989440","https://openalex.org/W2171278097","https://openalex.org/W2171965515","https://openalex.org/W2197782679","https://openalex.org/W2260880659","https://openalex.org/W2272251967","https://openalex.org/W2276569499","https://openalex.org/W2293765620","https://openalex.org/W2338705265","https://openalex.org/W2346641862","https://openalex.org/W2395937589","https://openalex.org/W2398514781","https://openalex.org/W2405546739","https://openalex.org/W2412666979","https://openalex.org/W2433809021","https://openalex.org/W2436001372","https://openalex.org/W2522741018","https://openalex.org/W2553582928","https://openalex.org/W2567558666","https://openalex.org/W2589038172","https://openalex.org/W2726137762","https://openalex.org/W2744578392","https://openalex.org/W2748181679","https://openalex.org/W2796300194","https://openalex.org/W2803638136","https://openalex.org/W2806875945","https://openalex.org/W2810116858","https://openalex.org/W2898349875","https://openalex.org/W2898720903","https://openalex.org/W2911964244","https://openalex.org/W2956054225","https://openalex.org/W3002599298","https://openalex.org/W3020014849","https://openalex.org/W3023482097","https://openalex.org/W3100608601","https://openalex.org/W3121257585","https://openalex.org/W4205646471","https://openalex.org/W4206684001","https://openalex.org/W4210764005","https://openalex.org/W4242883546","https://openalex.org/W4255655703","https://openalex.org/W6600827356","https://openalex.org/W6602348844","https://openalex.org/W6606412491","https://openalex.org/W6610117260","https://openalex.org/W6625168331","https://openalex.org/W6634395915","https://openalex.org/W6675354045","https://openalex.org/W6677712588","https://openalex.org/W6679792166","https://openalex.org/W6684172481","https://openalex.org/W6687705047","https://openalex.org/W6697452742","https://openalex.org/W6749044177","https://openalex.org/W6804477190","https://openalex.org/W7064393817"],"related_works":["https://openalex.org/W2384667405","https://openalex.org/W2389266787","https://openalex.org/W2363885542","https://openalex.org/W2387206255","https://openalex.org/W2355570285","https://openalex.org/W2364958354","https://openalex.org/W2352797113","https://openalex.org/W2372137583","https://openalex.org/W2374872591","https://openalex.org/W1548050717"],"abstract_inverted_index":{"Crowd":[0],"sourcing":[1],"and":[2,28,49,70,111,137,159,171,190,200],"human":[3,76],"computation":[4,77],"has":[5,56],"slowly":[6],"become":[7],"a":[8,58,81,139],"mainstay":[9],"for":[10,85,96,131,141],"many":[11],"application":[12],"areas":[13],"that":[14,105,112,121,179,191],"seek":[15],"to":[16,43,94,125,185],"leverage":[17],"the":[18,21,32,40,44,54,150],"crowd":[19,126],"in":[20,74],"development":[22],"of":[23,34,39,61,118,174,197],"high":[24],"quality":[25,62,98,119,157,181,205],"datasets,":[26],"annotations,":[27],"problem":[29],"solving":[30],"beyond":[31],"reach":[33],"current":[35],"AI":[36],"solutions.":[37,133],"One":[38],"major":[41],"challenges":[42],"domain":[45],"is":[46,88,108,115,129,183,193,207,214],"ensuring":[47],"high-quality":[48],"diligent":[50],"work.":[51],"In":[52,100],"response,":[53],"literature":[55],"seen":[57],"large":[59],"number":[60],"control":[63,120,158,182,206],"mechanisms":[64],"each":[65],"voicing":[66],"(sometimes":[67],"domain-specific)":[68],"benefits":[69],"advantages":[71],"when":[72],"deployed":[73],"largescale":[75],"projects.":[78],"This":[79],"creates":[80],"complex":[82],"design":[83],"space":[84],"practitioners:":[86],"it":[87,213],"not":[89,211],"always":[90],"clear":[91],"which":[92],"mechanism(s)":[93],"use":[95],"maximal":[97],"control.":[99],"this":[101,106,122,128,164,192],"article,":[102],"we":[103,143,148],"argue":[104],"decision":[107],"perhaps":[109],"overinflated":[110],"provided":[113],"there":[114],"\u201csome":[116],"kind\u201d":[117],"obviously":[123],"known":[124],"workers":[127],"sufficient":[130],"\u201chigh-quality\u201d":[132],"To":[134],"evidence":[135],"this,":[136],"provide":[138],"basis":[140],"discussion,":[142],"undertake":[144],"two":[145],"experiments":[146],"where":[147],"explore":[149],"relationship":[151],"between":[152],"task":[153,155],"design,":[154],"complexity,":[156],"solution":[160],"quality.":[161],"We":[162,177],"do":[163],"with":[165],"tasks":[166,196],"from":[167],"natural":[168],"language":[169],"processing,":[170],"image":[172],"recognition":[173],"varying":[175,198],"complexity.":[176],"illustrate":[178],"minimal":[180],"enough":[184],"repel":[186],"constantly":[187],"underperforming":[188],"contributors":[189],"constant":[194],"across":[195],"complexity":[199],"formats.":[201],"Our":[202],"key":[203],"takeaway:":[204],"necessary,":[208],"but":[209],"seemingly":[210],"how":[212],"implemented.":[215]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2023,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}