{"id":"https://openalex.org/W4220728222","doi":"https://doi.org/10.1145/3498366.3505815","title":"The Crowd is Made of People","display_name":"The Crowd is Made of People","publication_year":2022,"publication_date":"2022-03-12","ids":{"openalex":"https://openalex.org/W4220728222","doi":"https://doi.org/10.1145/3498366.3505815"},"language":"en","primary_location":{"id":"doi:10.1145/3498366.3505815","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3498366.3505815","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGIR Conference on Human Information Interaction and Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5046377370","display_name":"Paul Thomas","orcid":"https://orcid.org/0000-0003-2425-3136"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paul Thomas","raw_affiliation_strings":["Microsoft, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Australia","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078735102","display_name":"Gabriella Kazai","orcid":"https://orcid.org/0009-0002-5158-6630"},"institutions":[{"id":"https://openalex.org/I4210108625","display_name":"Microsoft (United Kingdom)","ror":"https://ror.org/01rw27z95","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210108625"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Gabriella Kazai","raw_affiliation_strings":["Microsoft, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, United Kingdom","institution_ids":["https://openalex.org/I4210108625"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076259865","display_name":"Ryen W. White","orcid":"https://orcid.org/0000-0002-0265-4249"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ryen White","raw_affiliation_strings":["Microsoft, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5055132321","display_name":"Nick Craswell","orcid":"https://orcid.org/0000-0002-9351-8137"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nick Craswell","raw_affiliation_strings":["Microsoft, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, United States","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.9255,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.95296703,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"25","last_page":"35"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11500","display_name":"Evacuation and Crowd Dynamics","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/2212","display_name":"Ocean Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.9474999904632568,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5628411769866943}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5628411769866943}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3498366.3505815","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3498366.3505815","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGIR Conference on Human Information Interaction and Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":55,"referenced_works":["https://openalex.org/W1497983610","https://openalex.org/W1522816040","https://openalex.org/W1541075651","https://openalex.org/W1951724000","https://openalex.org/W1968927634","https://openalex.org/W1978108654","https://openalex.org/W1987831316","https://openalex.org/W1998838557","https://openalex.org/W2007018772","https://openalex.org/W2008285050","https://openalex.org/W2008803468","https://openalex.org/W2013575152","https://openalex.org/W2022995284","https://openalex.org/W2051752522","https://openalex.org/W2051913671","https://openalex.org/W2062236692","https://openalex.org/W2071446250","https://openalex.org/W2077037584","https://openalex.org/W2094607109","https://openalex.org/W2097246133","https://openalex.org/W2098865355","https://openalex.org/W2107576603","https://openalex.org/W2129728332","https://openalex.org/W2132232498","https://openalex.org/W2133968343","https://openalex.org/W2140296984","https://openalex.org/W2153763032","https://openalex.org/W2166426462","https://openalex.org/W2273794225","https://openalex.org/W2395937589","https://openalex.org/W2397692773","https://openalex.org/W2405546739","https://openalex.org/W2461055567","https://openalex.org/W2519434959","https://openalex.org/W2582743722","https://openalex.org/W2582919740","https://openalex.org/W2593021276","https://openalex.org/W2774486220","https://openalex.org/W2783668259","https://openalex.org/W2783946743","https://openalex.org/W2811371717","https://openalex.org/W2891036412","https://openalex.org/W2895098505","https://openalex.org/W2913282099","https://openalex.org/W2921524648","https://openalex.org/W2941486823","https://openalex.org/W2948630599","https://openalex.org/W3005284662","https://openalex.org/W3045906875","https://openalex.org/W3125913709","https://openalex.org/W3212896246","https://openalex.org/W4242847335","https://openalex.org/W4255321213","https://openalex.org/W4292157289","https://openalex.org/W6629407803"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Like":[0],"many":[1],"other":[2],"researchers,":[3],"at":[4,73],"Microsoft":[5],"Bing":[6],"we":[7,62,67],"use":[8],"external":[9],"\u201ccrowd\u201d":[10],"judges":[11],"to":[12,22,45,53,89],"label":[13,100],"results":[14],"from":[15],"a":[16,99],"search":[17,81],"engine\u2014especially,":[18],"although":[19],"not":[20,132],"exclusively,":[21],"obtain":[23],"relevance":[24],"labels":[25,34],"for":[26,80],"offline":[27],"evaluation":[28,82,158],"in":[29,83,156],"the":[30,78,90],"Cranfield":[31],"tradition.":[32],"Crowdsourced":[33],"are":[35,43,121],"relatively":[36],"cheap,":[37],"and":[38,48,76,94,110,126,155],"hence":[39],"very":[40],"popular,":[41],"but":[42,136],"prone":[44],"disagreements,":[46],"spam,":[47],"various":[49],"biases":[50],"which":[51],"appear":[52],"be":[54,149],"unexplained":[55],"\u201cnoise\u201d":[56],"or":[57],"\u201cerror\u201d.":[58],"In":[59],"this":[60],"paper,":[61],"provide":[63],"examples":[64],"of":[65,92,96,139],"problems":[66],"have":[68],"encountered":[69],"running":[70],"crowd":[71],"labelling":[72],"large":[74],"scale":[75],"around":[77],"globe,":[79],"particular.":[84],"We":[85],"demonstrate":[86],"effects":[87,120],"due":[88],"time":[91],"day":[93,95],"week":[97],"that":[98,143],"is":[101,131,137],"given;":[102],"fatigue;":[103],"anchoring;":[104],"exposure;":[105],"left-side":[106],"bias;":[107],"task":[108],"switching;":[109],"simple":[111,117],"disagreement":[112],"between":[113],"judges.":[114],"Rather":[115],"than":[116],"\u201cerror\u201d,":[118],"these":[119],"consistent":[122],"with":[123],"well-known":[124],"physiological":[125],"cognitive":[127],"factors.":[128],"\u201cThe":[129],"crowd\u201d":[130],"some":[133],"abstract":[134],"machinery,":[135],"made":[138],"people.":[140],"Human":[141],"factors":[142],"affect":[144],"people\u2019s":[145],"judgement":[146],"behaviour":[147],"must":[148],"considered":[150],"when":[151],"designing":[152],"research":[153],"evaluations":[154],"interpreting":[157],"metrics.":[159]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":1}],"updated_date":"2026-07-30T17:31:21.811387","created_date":"2025-10-10T00:00:00"}