{"id":"https://openalex.org/W3209621813","doi":"https://doi.org/10.1145/3459637.3482032","title":"Aggregation Techniques in Crowdsourcing: Multiple Choice Questions and Beyond","display_name":"Aggregation Techniques in Crowdsourcing: Multiple Choice Questions and Beyond","publication_year":2021,"publication_date":"2021-10-26","ids":{"openalex":"https://openalex.org/W3209621813","doi":"https://doi.org/10.1145/3459637.3482032","mag":"3209621813"},"language":"en","primary_location":{"id":"doi:10.1145/3459637.3482032","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3459637.3482032","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM International Conference on Information & Knowledge Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088762737","display_name":"Djellel Difallah","orcid":"https://orcid.org/0000-0002-7513-6047"},"institutions":[{"id":"https://openalex.org/I120250893","display_name":"New York University Abu Dhabi","ror":"https://ror.org/00e5k0821","country_code":"AE","type":"education","lineage":["https://openalex.org/I120250893","https://openalex.org/I57206974"]}],"countries":["AE"],"is_corresponding":true,"raw_author_name":"Djellel Difallah","raw_affiliation_strings":["NYU Abu Dhabi, Abu Dhabi, UAE"],"affiliations":[{"raw_affiliation_string":"NYU Abu Dhabi, Abu Dhabi, UAE","institution_ids":["https://openalex.org/I120250893"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021006031","display_name":"Alessandro Checco","orcid":"https://orcid.org/0000-0002-0981-3409"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Alessandro Checco","raw_affiliation_strings":["University of Sheffield, Sheffield, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Sheffield, Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5088762737"],"corresponding_institution_ids":["https://openalex.org/I120250893"],"apc_list":null,"apc_paid":null,"fwci":1.1717,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.83000543,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"4842","last_page":"4844"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9793000221252441,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9337000250816345,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9740259051322937},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8261871337890625},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.7028347253799438},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6492182016372681},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6410278677940369},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.5840526819229126},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.552036702632904},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.48817235231399536},{"id":"https://openalex.org/keywords/majority-rule","display_name":"Majority rule","score":0.47469180822372437},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.46968868374824524},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.464758038520813},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.4424082934856415},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.43424105644226074},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4108119010925293},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.1829676628112793}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9740259051322937},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8261871337890625},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7028347253799438},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6492182016372681},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6410278677940369},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.5840526819229126},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.552036702632904},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48817235231399536},{"id":"https://openalex.org/C153668964","wikidata":"https://www.wikidata.org/wiki/Q27636","display_name":"Majority rule","level":2,"score":0.47469180822372437},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46968868374824524},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.464758038520813},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.4424082934856415},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.43424105644226074},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4108119010925293},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.1829676628112793},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3459637.3482032","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3459637.3482032","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM International Conference on Information & Knowledge Management","raw_type":"proceedings-article"},{"id":"pmh:oai:iris.uniroma1.it:11573/1696313","is_oa":false,"landing_page_url":"https://hdl.handle.net/11573/1696313","pdf_url":null,"source":{"id":"https://openalex.org/S4377196107","display_name":"IRIS Research product catalog (Sapienza University of Rome)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":36,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W172170795","https://openalex.org/W567643054","https://openalex.org/W1204558347","https://openalex.org/W1494186835","https://openalex.org/W1501025848","https://openalex.org/W1541280084","https://openalex.org/W2012668444","https://openalex.org/W2034771068","https://openalex.org/W2104814277","https://openalex.org/W2113878109","https://openalex.org/W2129345386","https://openalex.org/W2149273804","https://openalex.org/W2152009989","https://openalex.org/W2161816739","https://openalex.org/W2164545125","https://openalex.org/W2210040668","https://openalex.org/W2250977991","https://openalex.org/W2294447324","https://openalex.org/W2404698280","https://openalex.org/W2585226541","https://openalex.org/W2605991684","https://openalex.org/W2765320432","https://openalex.org/W2780163489","https://openalex.org/W2786900223","https://openalex.org/W2807760453","https://openalex.org/W2907800194","https://openalex.org/W2911761300","https://openalex.org/W2964067789","https://openalex.org/W2966836724","https://openalex.org/W2974290810","https://openalex.org/W2998598262","https://openalex.org/W3001315432","https://openalex.org/W3029299514","https://openalex.org/W3038629767","https://openalex.org/W3047395298"],"related_works":["https://openalex.org/W2178171640","https://openalex.org/W2245294024","https://openalex.org/W1472755691","https://openalex.org/W2963660546","https://openalex.org/W2055733113","https://openalex.org/W3102460643","https://openalex.org/W2018369380","https://openalex.org/W2522741018","https://openalex.org/W2938213904","https://openalex.org/W2109094787"],"abstract_inverted_index":{"Crowdsourcing":[0],"has":[1],"been":[2,155],"leveraged":[3],"in":[4,16,211],"various":[5],"tasks":[6],"and":[7,44,48,57,68,172,185],"applications,":[8],"primarily":[9],"to":[10,55,89,109,127,138,169,203,209],"gather":[11],"information":[12],"from":[13,38,195],"human":[14],"annotators":[15],"exchange":[17],"for":[18,157,177],"a":[19,74,85,90],"monetary":[20],"reward.":[21],"The":[22],"main":[23],"challenge":[24],"associated":[25],"with":[26,111],"crowdsourcing":[27,158,213],"is":[28],"the":[29,33,80,105,125,140,191,196,206],"low":[30],"quality":[31,52,101],"of":[32,152,163],"results,":[34],"which":[35],"can":[36,50,83],"stem":[37],"multiple":[39,128],"reasons,":[40],"including":[41],"bias,":[42],"error,":[43],"adversarial":[45],"behavior.":[46],"Researchers":[47],"practitioners":[49],"apply":[51,210],"control":[53,102],"methods":[54,65,103,119],"prevent":[56],"detect":[58],"low-quality":[59],"responses.":[60],"For":[61],"example,":[62],"worker":[63],"selection":[64],"utilize":[66],"qualifications":[67],"attention":[69],"check":[70],"questions":[71],"before":[72],"assigning":[73,124],"task.":[75],"Similarly,":[76],"task":[77,92,121,126,164],"routing":[78],"identifies":[79],"workers":[81],"who":[82],"provide":[84],"more":[86],"accurate":[87],"response":[88],"given":[91],"type":[93],"using":[94],"recommender":[95],"system":[96],"techniques.":[97],"In":[98],"practice,":[99],"posterior":[100],"are":[104,116],"most":[106,141],"common":[107,171],"approach":[108],"deal":[110],"noisy":[112],"labels":[113],"once":[114],"they":[115],"obtained.":[117],"Such":[118],"require":[120],"repetition,":[122],"i.e.,":[123],"crowd-workers,":[129],"followed":[130],"by":[131],"an":[132,146],"aggregation":[133,159,175],"mechanism":[134],"(aka":[135],"truth":[136],"inference)":[137],"select":[139],"likely":[142],"answer":[143],"or":[144],"request":[145],"additional":[147],"label.":[148],"A":[149],"large":[150],"number":[151],"techniques":[153,176,208],"have":[154],"proposed":[156],"covering":[160],"several":[161],"types":[162],"types.":[165],"This":[166],"tutorial":[167],"aims":[168],"present":[170],"recent":[173],"label":[174],"multiple-choice":[178],"questions,":[179],"multi-class":[180],"labels,":[181],"ratings,":[182],"pairwise":[183],"comparison,":[184],"image/text":[186],"annotation.":[187],"We":[188],"believe":[189],"that":[190],"audience":[192],"will":[193],"benefit":[194],"focus":[197],"on":[198],"this":[199],"specific":[200],"research":[201],"area":[202],"learn":[204],"about":[205],"best":[207],"their":[212],"projects.":[214]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2021-11-08T00:00:00"}