{"id":"https://openalex.org/W4292341621","doi":"https://doi.org/10.1145/3148148","title":"Quality Control in Crowdsourcing","display_name":"Quality Control in Crowdsourcing","publication_year":2018,"publication_date":"2018-01-04","ids":{"openalex":"https://openalex.org/W4292341621","doi":"https://doi.org/10.1145/3148148"},"language":"en","primary_location":{"id":"doi:10.1145/3148148","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3148148","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"type":"review","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076749458","display_name":"Florian Daniel","orcid":"https://orcid.org/0000-0003-3004-8702"},"institutions":[{"id":"https://openalex.org/I93860229","display_name":"Politecnico di Milano","ror":"https://ror.org/01nffqt88","country_code":"IT","type":"education","lineage":["https://openalex.org/I93860229"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Florian Daniel","raw_affiliation_strings":["Politecnico di Milano, Milano, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Politecnico di Milano, Milano, Italy","institution_ids":["https://openalex.org/I93860229"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085564474","display_name":"Pavel Kucherbaev","orcid":null},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Pavel Kucherbaev","raw_affiliation_strings":["Delft University of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Delft University of Technology","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063621424","display_name":"Cinzia Cappiello","orcid":"https://orcid.org/0000-0001-6062-5174"},"institutions":[{"id":"https://openalex.org/I93860229","display_name":"Politecnico di Milano","ror":"https://ror.org/01nffqt88","country_code":"IT","type":"education","lineage":["https://openalex.org/I93860229"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Cinzia Cappiello","raw_affiliation_strings":["Politecnico di Milano, Milano, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Politecnico di Milano, Milano, Italy","institution_ids":["https://openalex.org/I93860229"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042705320","display_name":"Boualem Benatallah","orcid":"https://orcid.org/0000-0002-8805-1130"},"institutions":[{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Boualem Benatallah","raw_affiliation_strings":["University of New South Wales, CSE, Sydney, NSW, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of New South Wales, CSE, Sydney, NSW, Australia","institution_ids":["https://openalex.org/I31746571"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046739900","display_name":"Mohammad Allahbakhsh","orcid":"https://orcid.org/0000-0002-2861-7745"},"institutions":[{"id":"https://openalex.org/I112450260","display_name":"Zabol University","ror":"https://ror.org/03d9mz263","country_code":"IR","type":"education","lineage":["https://openalex.org/I112450260"]}],"countries":["IR"],"is_corresponding":false,"raw_author_name":"Mohammad Allahbakhsh","raw_affiliation_strings":["University of Zabol, Zabol, Iran"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Zabol, Zabol, Iran","institution_ids":["https://openalex.org/I112450260"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5076749458"],"corresponding_institution_ids":["https://openalex.org/I93860229"],"apc_list":null,"apc_paid":null,"fwci":51.5721,"has_fulltext":false,"cited_by_count":306,"citation_normalized_percentile":{"value":0.99924234,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"51","issue":"1","first_page":"1","last_page":"40"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9919000267982483,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9863268733024597},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8008180856704712},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.7530211806297302},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.6616908311843872},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6420082449913025},{"id":"https://openalex.org/keywords/crowdsourcing-software-development","display_name":"Crowdsourcing software development","score":0.6168871521949768},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.5796395540237427},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.4941242039203644},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.4039114713668823},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3131391406059265},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.20583897829055786}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9863268733024597},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8008180856704712},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.7530211806297302},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.6616908311843872},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6420082449913025},{"id":"https://openalex.org/C32810761","wikidata":"https://www.wikidata.org/wiki/Q17082146","display_name":"Crowdsourcing software development","level":5,"score":0.6168871521949768},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.5796395540237427},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4941242039203644},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.4039114713668823},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3131391406059265},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.20583897829055786},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.0},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3148148","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3148148","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320334704","display_name":"Australian Research Council","ror":"https://ror.org/05mmh0f86"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":211,"referenced_works":["https://openalex.org/W68991936","https://openalex.org/W108763474","https://openalex.org/W110548153","https://openalex.org/W118259468","https://openalex.org/W187985411","https://openalex.org/W279631868","https://openalex.org/W960037631","https://openalex.org/W1181125918","https://openalex.org/W1482974529","https://openalex.org/W1487711117","https://openalex.org/W1494630173","https://openalex.org/W1499884756","https://openalex.org/W1501025848","https://openalex.org/W1504472146","https://openalex.org/W1510804179","https://openalex.org/W1518366186","https://openalex.org/W1518435216","https://openalex.org/W1536374875","https://openalex.org/W1539473175","https://openalex.org/W1543648998","https://openalex.org/W1565732937","https://openalex.org/W1573900212","https://openalex.org/W1595524969","https://openalex.org/W1633675443","https://openalex.org/W1641003075","https://openalex.org/W1643013964","https://openalex.org/W1720782345","https://openalex.org/W1813736802","https://openalex.org/W1815682670","https://openalex.org/W1818395637","https://openalex.org/W1846312067","https://openalex.org/W1854214752","https://openalex.org/W1881399225","https://openalex.org/W1903617842","https://openalex.org/W1963919160","https://openalex.org/W1969110305","https://openalex.org/W1969579764","https://openalex.org/W1971937094","https://openalex.org/W1979483312","https://openalex.org/W1985816302","https://openalex.org/W1988511622","https://openalex.org/W1991660937","https://openalex.org/W1992462783","https://openalex.org/W1995211436","https://openalex.org/W1997504572","https://openalex.org/W1999308248","https://openalex.org/W2000727220","https://openalex.org/W2001816027","https://openalex.org/W2002982773","https://openalex.org/W2004291985","https://openalex.org/W2005665592","https://openalex.org/W2009286572","https://openalex.org/W2014100104","https://openalex.org/W2018897815","https://openalex.org/W2021480189","https://openalex.org/W2022710553","https://openalex.org/W2024166834","https://openalex.org/W2024559213","https://openalex.org/W2024602912","https://openalex.org/W2025627988","https://openalex.org/W2027600744","https://openalex.org/W2028953510","https://openalex.org/W2030793160","https://openalex.org/W2034078512","https://openalex.org/W2036431079","https://openalex.org/W2045114813","https://openalex.org/W2048167030","https://openalex.org/W2049774111","https://openalex.org/W2050491563","https://openalex.org/W2050867684","https://openalex.org/W2052756509","https://openalex.org/W2055090888","https://openalex.org/W2058738785","https://openalex.org/W2058892254","https://openalex.org/W2061093454","https://openalex.org/W2062742763","https://openalex.org/W2063606769","https://openalex.org/W2072720670","https://openalex.org/W2074275752","https://openalex.org/W2077119967","https://openalex.org/W2078883237","https://openalex.org/W2080226267","https://openalex.org/W2085295345","https://openalex.org/W2088664555","https://openalex.org/W2096519149","https://openalex.org/W2097246133","https://openalex.org/W2099578472","https://openalex.org/W2102271161","https://openalex.org/W2104134129","https://openalex.org/W2106675345","https://openalex.org/W2107394329","https://openalex.org/W2107756083","https://openalex.org/W2111298664","https://openalex.org/W2113733890","https://openalex.org/W2113805011","https://openalex.org/W2114100913","https://openalex.org/W2114574011","https://openalex.org/W2115206325","https://openalex.org/W2116664070","https://openalex.org/W2116795783","https://openalex.org/W2116915306","https://openalex.org/W2117470435","https://openalex.org/W2118153153","https://openalex.org/W2122815602","https://openalex.org/W2122885251","https://openalex.org/W2124503193","https://openalex.org/W2124582642","https://openalex.org/W2124793952","https://openalex.org/W2124994029","https://openalex.org/W2126079815","https://openalex.org/W2127008633","https://openalex.org/W2129034804","https://openalex.org/W2131141188","https://openalex.org/W2131864341","https://openalex.org/W2132481689","https://openalex.org/W2132962756","https://openalex.org/W2133485007","https://openalex.org/W2135504089","https://openalex.org/W2136150876","https://openalex.org/W2136222780","https://openalex.org/W2139977214","https://openalex.org/W2140287075","https://openalex.org/W2140890285","https://openalex.org/W2146928171","https://openalex.org/W2148479118","https://openalex.org/W2148517649","https://openalex.org/W2148527862","https://openalex.org/W2150585298","https://openalex.org/W2150597317","https://openalex.org/W2151401338","https://openalex.org/W2153075039","https://openalex.org/W2156011021","https://openalex.org/W2158375896","https://openalex.org/W2159587809","https://openalex.org/W2160338921","https://openalex.org/W2160370399","https://openalex.org/W2163522723","https://openalex.org/W2166862287","https://openalex.org/W2167913131","https://openalex.org/W2168071746","https://openalex.org/W2168396085","https://openalex.org/W2168765606","https://openalex.org/W2169036685","https://openalex.org/W2169882833","https://openalex.org/W2170493558","https://openalex.org/W2171450492","https://openalex.org/W2175852750","https://openalex.org/W2177901171","https://openalex.org/W2178171640","https://openalex.org/W2181558882","https://openalex.org/W2182472765","https://openalex.org/W2187291759","https://openalex.org/W2187643071","https://openalex.org/W2221430606","https://openalex.org/W2228844318","https://openalex.org/W2248774094","https://openalex.org/W2252446605","https://openalex.org/W2258849743","https://openalex.org/W2274963131","https://openalex.org/W2279346661","https://openalex.org/W2284509824","https://openalex.org/W2289052659","https://openalex.org/W2290009368","https://openalex.org/W2292342791","https://openalex.org/W2293767674","https://openalex.org/W2294447324","https://openalex.org/W2296739333","https://openalex.org/W2335974056","https://openalex.org/W2339482357","https://openalex.org/W2339954278","https://openalex.org/W2395348601","https://openalex.org/W2398690976","https://openalex.org/W2400269077","https://openalex.org/W2402489522","https://openalex.org/W2402582321","https://openalex.org/W2404777434","https://openalex.org/W2405546739","https://openalex.org/W2408493841","https://openalex.org/W2408524209","https://openalex.org/W2410023545","https://openalex.org/W2460107691","https://openalex.org/W2467989257","https://openalex.org/W2472528732","https://openalex.org/W2473634138","https://openalex.org/W2512053137","https://openalex.org/W2534774337","https://openalex.org/W2536952838","https://openalex.org/W2540714355","https://openalex.org/W2550444944","https://openalex.org/W2558888286","https://openalex.org/W2567558666","https://openalex.org/W2584306408","https://openalex.org/W2599032673","https://openalex.org/W2950020305","https://openalex.org/W2952450321","https://openalex.org/W2952598059","https://openalex.org/W2962755741","https://openalex.org/W2963284842","https://openalex.org/W3098243577","https://openalex.org/W3102460643","https://openalex.org/W3103797191","https://openalex.org/W3104998832","https://openalex.org/W3121257585","https://openalex.org/W3123168165","https://openalex.org/W3124258878","https://openalex.org/W3125141063","https://openalex.org/W4297685397","https://openalex.org/W4313047369","https://openalex.org/W6712662495","https://openalex.org/W6821635801","https://openalex.org/W6837117956"],"related_works":["https://openalex.org/W2512762878","https://openalex.org/W162737300","https://openalex.org/W2520748299","https://openalex.org/W2033745466","https://openalex.org/W2520160102","https://openalex.org/W2011363905","https://openalex.org/W2046555014","https://openalex.org/W3131060711","https://openalex.org/W2548342535","https://openalex.org/W3125008079"],"abstract_inverted_index":{"Crowdsourcing":[0],"enables":[1],"one":[2],"to":[3,106,112,139],"leverage":[4],"on":[5,180],"the":[6,72,97,114,118,131,141,144,147,167,170],"intelligence":[7],"and":[8,35,81,89,108,111,133,146,149,154,176],"wisdom":[9],"of":[10,14,53,68,77,99,117,143,160,169],"potentially":[11],"large":[12],"groups":[13],"individuals":[15],"toward":[16],"solving":[17],"problems.":[18,157],"Common":[19],"problems":[20],"approached":[21],"with":[22,79],"crowdsourcing":[23,100,128],"are":[24,40,164],"labeling":[25],"images,":[26],"translating":[27],"or":[28,33,44],"transcribing":[29],"text,":[30],"providing":[31],"opinions":[32],"ideas,":[34],"similar\u2014all":[36],"tasks":[37],"that":[38,135,151],"computers":[39],"not":[41],"good":[42],"at":[43],"where":[45],"they":[46],"may":[47],"even":[48],"fail":[49],"altogether.":[50],"The":[51],"introduction":[52],"humans":[54],"into":[55],"computations":[56],"and/or":[57],"everyday":[58],"work,":[59],"however,":[60],"also":[61],"poses":[62],"critical,":[63],"novel":[64],"challenges":[65],"in":[66,96],"terms":[67],"quality":[69,95,125,156],"control,":[70],"as":[71,105],"crowd":[73],"is":[74],"typically":[75],"composed":[76],"people":[78],"unknown":[80],"very":[82],"diverse":[83],"abilities,":[84],"skills,":[85],"interests,":[86],"personal":[87],"objectives,":[88],"technological":[90],"resources.":[91],"This":[92],"survey":[93,122],"studies":[94],"context":[98],"along":[101],"several":[102],"dimensions,":[103],"so":[104],"define":[107],"characterize":[109],"it":[110],"understand":[113],"current":[115],"state":[116,168],"art.":[119],"Specifically,":[120],"this":[121],"derives":[123],"a":[124],"model":[126],"for":[127],"tasks,":[129],"identifies":[130,173],"methods":[132],"techniques":[134],"can":[136],"be":[137],"used":[138],"assess":[140],"attributes":[142],"model,":[145],"actions":[148],"strategies":[150],"help":[152],"prevent":[153],"mitigate":[155],"An":[158],"analysis":[159],"how":[161],"these":[162],"features":[163],"supported":[165],"by":[166],"art":[171],"further":[172],"open":[174],"issues":[175],"informs":[177],"an":[178],"outlook":[179],"hot":[181],"future":[182],"research":[183],"directions.":[184]},"counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":30},{"year":2024,"cited_by_count":41},{"year":2023,"cited_by_count":35},{"year":2022,"cited_by_count":31},{"year":2021,"cited_by_count":66},{"year":2020,"cited_by_count":41},{"year":2019,"cited_by_count":38},{"year":2018,"cited_by_count":11}],"updated_date":"2026-05-24T08:33:08.758527","created_date":"2022-08-19T00:00:00"}