{"id":"https://openalex.org/W2809759799","doi":"https://doi.org/10.1145/3209978.3210033","title":"Crowd vs. Expert","display_name":"Crowd vs. Expert","publication_year":2018,"publication_date":"2018-06-27","ids":{"openalex":"https://openalex.org/W2809759799","doi":"https://doi.org/10.1145/3209978.3210033","mag":"2809759799"},"language":"en","primary_location":{"id":"doi:10.1145/3209978.3210033","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3209978.3210033","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086826049","display_name":"M\u00fccahid Kutlu","orcid":"https://orcid.org/0000-0002-5660-4992"},"institutions":[{"id":"https://openalex.org/I60342839","display_name":"Qatar University","ror":"https://ror.org/00yhnba62","country_code":"QA","type":"education","lineage":["https://openalex.org/I60342839"]}],"countries":["QA"],"is_corresponding":false,"raw_author_name":"Mucahid Kutlu","raw_affiliation_strings":["Qatar University, Doha, Qatar"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Qatar University, Doha, Qatar","institution_ids":["https://openalex.org/I60342839"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049044903","display_name":"Tyler McDonnell","orcid":null},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tyler McDonnell","raw_affiliation_strings":["University of Texas at Austin, Austin, TX, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Texas at Austin, Austin, TX, USA","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045697035","display_name":"Yassmine Barkallah","orcid":null},"institutions":[{"id":"https://openalex.org/I60342839","display_name":"Qatar University","ror":"https://ror.org/00yhnba62","country_code":"QA","type":"education","lineage":["https://openalex.org/I60342839"]}],"countries":["QA"],"is_corresponding":false,"raw_author_name":"Yassmine Barkallah","raw_affiliation_strings":["Qatar University, Doha, Qatar"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Qatar University, Doha, Qatar","institution_ids":["https://openalex.org/I60342839"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032141720","display_name":"Tamer Elsayed","orcid":"https://orcid.org/0000-0001-5786-4668"},"institutions":[{"id":"https://openalex.org/I60342839","display_name":"Qatar University","ror":"https://ror.org/00yhnba62","country_code":"QA","type":"education","lineage":["https://openalex.org/I60342839"]}],"countries":["QA"],"is_corresponding":false,"raw_author_name":"Tamer Elsayed","raw_affiliation_strings":["Qatar University, Doha, Qatar"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Qatar University, Doha, Qatar","institution_ids":["https://openalex.org/I60342839"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023397430","display_name":"Matthew Lease","orcid":"https://orcid.org/0000-0002-0056-2834"},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew Lease","raw_affiliation_strings":["University of Texas at Austin, Austin, TX, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Texas at Austin, Austin, TX, USA","institution_ids":["https://openalex.org/I86519309"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":19,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"805","last_page":"814"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8676114678382874},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6765382885932922},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6424167156219482},{"id":"https://openalex.org/keywords/transparency","display_name":"Transparency (behavior)","score":0.6093150973320007},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.5823293924331665},{"id":"https://openalex.org/keywords/taxonomy","display_name":"Taxonomy (biology)","score":0.5149998664855957},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.501915454864502},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4740833044052124},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.45293691754341125},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.34227049350738525},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.253378301858902},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.20696431398391724}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8676114678382874},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6765382885932922},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6424167156219482},{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.6093150973320007},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.5823293924331665},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.5149998664855957},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.501915454864502},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4740833044052124},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.45293691754341125},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.34227049350738525},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.253378301858902},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.20696431398391724},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3209978.3210033","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3209978.3210033","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:qspace.qu.edu.qa:10576/11838","is_oa":false,"landing_page_url":"http://hdl.handle.net/10576/11838","pdf_url":null,"source":{"id":"https://openalex.org/S4306400014","display_name":"Qatar University QSpace (Qatar University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I60342839","host_organization_name":"Qatar University","host_organization_lineage":["https://openalex.org/I60342839"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309815","display_name":"Qatar Foundation","ror":"https://ror.org/01cawbq05"},{"id":"https://openalex.org/F4320332178","display_name":"National Institute of Standards and Technology","ror":"https://ror.org/05xpvk416"},{"id":"https://openalex.org/F4320332753","display_name":"Qatar National Research Fund","ror":"https://ror.org/01svaqq28"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W112254115","https://openalex.org/W179309874","https://openalex.org/W1966835268","https://openalex.org/W1996422672","https://openalex.org/W2008285050","https://openalex.org/W2013575152","https://openalex.org/W2039017419","https://openalex.org/W2050467853","https://openalex.org/W2053528567","https://openalex.org/W2062236692","https://openalex.org/W2071446250","https://openalex.org/W2094607109","https://openalex.org/W2107946802","https://openalex.org/W2119461458","https://openalex.org/W2120308175","https://openalex.org/W2137811636","https://openalex.org/W2139516750","https://openalex.org/W2150234222","https://openalex.org/W2153763032","https://openalex.org/W2165062189","https://openalex.org/W2170493558","https://openalex.org/W2327405240","https://openalex.org/W2388783363","https://openalex.org/W2558888286","https://openalex.org/W2811107472","https://openalex.org/W2963929562"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W1503094549","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W2337920774","https://openalex.org/W4318823662","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W3207526114","https://openalex.org/W1497983610"],"abstract_inverted_index":{"While":[0],"crowdsourcing":[1],"offers":[2],"a":[3,32,157],"low-cost,":[4],"scalable":[5],"way":[6],"to":[7,40,59,104,118,185],"collect":[8],"relevance":[9,46],"judgments,":[10],"lack":[11],"of":[12,24,34,116,142,150,165],"transparency":[13],"with":[14,179,203],"remote":[15],"crowd":[16,38,72,82,99,130,173,201],"work":[17],"has":[18],"limited":[19],"understanding":[20],"about":[21],"the":[22,64,114,147,162,189],"quality":[23],"collected":[25],"judgments.":[26],"In":[27,49],"prior":[28],"work,":[29,51],"we":[30,52,96,155,167],"showed":[31],"variety":[33],"benefits":[35],"from":[36],"asking":[37],"workers":[39],"provide":[41],"\\em":[42],"rationales":[43,117],"for":[44,74,109,124,160,210],"each":[45,151,211],"judgment":[47],"\\citemcdonnell2016relevant.":[48],"this":[50,92],"scale":[53],"up":[54],"our":[55,196],"rationale-based":[56],"judging":[57,125,186,212],"design":[58],"assess":[60],"its":[61],"reliability":[62],"on":[63,122],"2014":[65],"TREC":[66,191],"Web":[67],"Track,":[68],"collecting":[69],"roughly":[70],"25K":[71],"judgments":[73,100,202],"5K":[75],"document-topic":[76],"pairs.":[77],"We":[78,111,169,193],"also":[79,194],"study":[80],"having":[81],"judges":[83],"perform":[84],"topic-focused":[85],"judging,":[86],"rather":[87],"than":[88],"across":[89],"topics,":[90],"finding":[91],"improves":[93],"quality.":[94],"Overall,":[95],"show":[97,170],"that":[98,171],"can":[101],"be":[102],"used":[103],"reliably":[105],"rank":[106],"IR":[107],"systems":[108],"evaluation.":[110],"further":[112],"explore":[113],"potential":[115],"shed":[119],"new":[120,158],"light":[121],"reasons":[123],"disagreement":[126,152,166,180,213],"between":[127],"experts":[128],"and":[129,134,154,177,205],"workers.":[131],"Our":[132],"qualitative":[133],"quantitative":[135],"analysis":[136],"distinguishes":[137],"subjective":[138],"vs.\\":[139],"objective":[140],"forms":[141],"disagreement,":[143],"as":[144,146],"well":[145],"relative":[148],"importance":[149],"cause,":[153],"present":[156],"taxonomy":[159,207],"organizing":[161],"different":[163],"types":[164],"observe.":[168],"many":[172,182],"disagreements":[174],"seem":[175],"valid":[176],"plausible,":[178],"in":[181],"cases":[183],"due":[184],"errors":[187],"by":[188],"original":[190],"assessors.":[192],"share":[195],"WebCrowd25k":[197],"dataset,":[198],"including:":[199],"(1)":[200],"rationales,":[204],"(2)":[206],"category":[208],"labels":[209],"analyzed.":[214]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2018-07-10T00:00:00"}