{"id":"https://openalex.org/W7165687347","doi":"https://doi.org/10.1145/3805689.3806510","title":"The Consensus Trap: Dissecting Subjectivity and the \u201cGround Truth\u201d Illusion in Data Annotation","display_name":"The Consensus Trap: Dissecting Subjectivity and the \u201cGround Truth\u201d Illusion in Data Annotation","publication_year":2026,"publication_date":"2026-06-23","ids":{"openalex":"https://openalex.org/W7165687347","doi":"https://doi.org/10.1145/3805689.3806510"},"language":null,"primary_location":{"id":"doi:10.1145/3805689.3806510","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805689.3806510","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805689.3806510","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114633693","display_name":"Sheza Munir","orcid":null},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Sheza Munir","raw_affiliation_strings":["Computer Science, University of Toronto, Toronto, Ontario, Canada"],"raw_orcid":"https://orcid.org/0009-0008-2175-254X","affiliations":[{"raw_affiliation_string":"Computer Science, University of Toronto, Toronto, Ontario, Canada","institution_ids":["https://openalex.org/I185261750"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125943637","display_name":"Benjamin Mah","orcid":null},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Benjamin Mah","raw_affiliation_strings":["Engineering Science, University of Toronto, Toronto, Ontario, Canada"],"raw_orcid":"https://orcid.org/0009-0003-9885-3799","affiliations":[{"raw_affiliation_string":"Engineering Science, University of Toronto, Toronto, Ontario, Canada","institution_ids":["https://openalex.org/I185261750"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125976407","display_name":"Krisha Kalsi","orcid":null},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Krisha Kalsi","raw_affiliation_strings":["Computer Science, University of Toronto, Toronto, Ontario, Canada"],"raw_orcid":"https://orcid.org/0009-0008-9640-0354","affiliations":[{"raw_affiliation_string":"Computer Science, University of Toronto, Toronto, Ontario, Canada","institution_ids":["https://openalex.org/I185261750"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007266018","display_name":"Shivani Kapania","orcid":"https://orcid.org/0000-0002-0152-4311"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shivani Kapania","raw_affiliation_strings":["School of Computer Science, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"],"raw_orcid":"https://orcid.org/0000-0002-0152-4311","affiliations":[{"raw_affiliation_string":"School of Computer Science, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024078157","display_name":"Julian Posada","orcid":"https://orcid.org/0000-0002-3285-6503"},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Julian Posada","raw_affiliation_strings":["American Studies, Yale University, New Haven, Connecticut, USA"],"raw_orcid":"https://orcid.org/0000-0002-3285-6503","affiliations":[{"raw_affiliation_string":"American Studies, Yale University, New Haven, Connecticut, USA","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058482884","display_name":"Edith Law","orcid":"https://orcid.org/0000-0002-7540-075X"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Edith Law","raw_affiliation_strings":["Computer Science, University of Waterloo, Waterloo, Ontario, Canada"],"raw_orcid":"https://orcid.org/0000-0002-7540-075X","affiliations":[{"raw_affiliation_string":"Computer Science, University of Waterloo, Waterloo, Ontario, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052316777","display_name":"Ding Wang","orcid":"https://orcid.org/0000-0002-5037-5017"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ding Wang","raw_affiliation_strings":["Google, Mountain View, California, USA"],"raw_orcid":"https://orcid.org/0000-0002-5037-5017","affiliations":[{"raw_affiliation_string":"Google, Mountain View, California, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5089574660","display_name":"Syed Ishtiaque Ahmed","orcid":"https://orcid.org/0000-0003-2452-0687"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Syed Ishtiaque Ahmed","raw_affiliation_strings":["Computer Science, University of Toronto, Toronto, Ontario, Canada"],"raw_orcid":"https://orcid.org/0000-0003-2452-0687","affiliations":[{"raw_affiliation_string":"Computer Science, University of Toronto, Toronto, Ontario, Canada","institution_ids":["https://openalex.org/I185261750"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2643","last_page":"2680"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12377","display_name":"Digital Humanities and Scholarship","score":0.066600002348423,"subfield":{"id":"https://openalex.org/subfields/1208","display_name":"Literature and Literary Theory"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12377","display_name":"Digital Humanities and Scholarship","score":0.066600002348423,"subfield":{"id":"https://openalex.org/subfields/1208","display_name":"Literature and Literary Theory"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.029500000178813934,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10258","display_name":"Philosophy and Theoretical Science","score":0.021299999207258224,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/subjectivity","display_name":"Subjectivity","score":0.5511999726295471},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5113000273704529},{"id":"https://openalex.org/keywords/illusion","display_name":"Illusion","score":0.3984000086784363},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.30730000138282776},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.27649998664855957}],"concepts":[{"id":"https://openalex.org/C202889954","wikidata":"https://www.wikidata.org/wiki/Q1139554","display_name":"Subjectivity","level":2,"score":0.5511999726295471},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5113000273704529},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.48739999532699585},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4779999852180481},{"id":"https://openalex.org/C184047640","wikidata":"https://www.wikidata.org/wiki/Q182593","display_name":"Illusion","level":2,"score":0.3984000086784363},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3546999990940094},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.30730000138282776},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.302700012922287},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.30160000920295715},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2840000092983246},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.2782999873161316},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.27649998664855957},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.2635999917984009}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3805689.3806510","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805689.3806510","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3805689.3806510","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805689.3806510","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities","score":0.4767642617225647}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":316,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W2926526813","https://openalex.org/W2969330194","https://openalex.org/W2982521860","https://openalex.org/W3002599298","https://openalex.org/W3004288302","https://openalex.org/W3004453146","https://openalex.org/W3009312460","https://openalex.org/W3029097580","https://openalex.org/W3029264758","https://openalex.org/W3029900269","https://openalex.org/W3030188676","https://openalex.org/W3030406212","https://openalex.org/W3031395001","https://openalex.org/W3031679542","https://openalex.org/W3032561420","https://openalex.org/W3032825225","https://openalex.org/W3034325332","https://openalex.org/W3034718094","https://openalex.org/W3035052826","https://openalex.org/W3035308065","https://openalex.org/W3037831233","https://openalex.org/W3047840587","https://openalex.org/W3080732897","https://openalex.org/W3093541323","https://openalex.org/W3093582825","https://openalex.org/W3094043653","https://openalex.org/W3094462788","https://openalex.org/W3112421345","https://openalex.org/W3118813946","https://openalex.org/W3119396809","https://openalex.org/W3120532656","https://openalex.org/W3120931376","https://openalex.org/W3123340107","https://openalex.org/W3123744018","https://openalex.org/W3124514224","https://openalex.org/W3127678221","https://openalex.org/W3128140567","https://openalex.org/W3133569056","https://openalex.org/W3133631714","https://openalex.org/W3133752603","https://openalex.org/W3134595736","https://openalex.org/W3135371071","https://openalex.org/W3138698152","https://openalex.org/W3157782429","https://openalex.org/W3157831956","https://openalex.org/W3158612374","https://openalex.org/W3158839228","https://openalex.org/W3159003756","https://openalex.org/W3160039929","https://openalex.org/W3160343688","https://openalex.org/W3160520794","https://openalex.org/W3160694721","https://openalex.org/W3160857525","https://openalex.org/W3160909372","https://openalex.org/W3161273899","https://openalex.org/W3161404450","https://openalex.org/W3161693913","https://openalex.org/W3161697659","https://openalex.org/W3161934537","https://openalex.org/W3162209985","https://openalex.org/W3162433157","https://openalex.org/W3163078977","https://openalex.org/W3163419709","https://openalex.org/W3163469193","https://openalex.org/W3173507217","https://openalex.org/W3174429158","https://openalex.org/W3174445427","https://openalex.org/W3174451219","https://openalex.org/W3177005832","https://openalex.org/W3177058128","https://openalex.org/W3177173791","https://openalex.org/W3187437888","https://openalex.org/W3189849087","https://openalex.org/W3192254170","https://openalex.org/W3193190893","https://openalex.org/W3196731672","https://openalex.org/W3199894449","https://openalex.org/W3203959757","https://openalex.org/W3205067036","https://openalex.org/W3205769026","https://openalex.org/W3205934028","https://openalex.org/W3206015227","https://openalex.org/W3206213635","https://openalex.org/W3206271843","https://openalex.org/W3206428286","https://openalex.org/W3207305517","https://openalex.org/W3212327893","https://openalex.org/W4221045135","https://openalex.org/W4224940987","https://openalex.org/W4224981752","https://openalex.org/W4224990314","https://openalex.org/W4224993409","https://openalex.org/W4225001019","https://openalex.org/W4225006216","https://openalex.org/W4225008476","https://openalex.org/W4225080590","https://openalex.org/W4225083500","https://openalex.org/W4225086340","https://openalex.org/W4225088745","https://openalex.org/W4225096151","https://openalex.org/W4225107257","https://openalex.org/W4225109361","https://openalex.org/W4225117586","https://openalex.org/W4225162252","https://openalex.org/W4225768055","https://openalex.org/W4225851417","https://openalex.org/W4226053008","https://openalex.org/W4226412112","https://openalex.org/W4280528151","https://openalex.org/W4280554323","https://openalex.org/W4280595958","https://openalex.org/W4282813546","https://openalex.org/W4283155630","https://openalex.org/W4283157665","https://openalex.org/W4283157985","https://openalex.org/W4283161771","https://openalex.org/W4283165820","https://openalex.org/W4283166427","https://openalex.org/W4283168936","https://openalex.org/W4283170948","https://openalex.org/W4288083800","https://openalex.org/W4288083802","https://openalex.org/W4288083803","https://openalex.org/W4306405418","https://openalex.org/W4308346016","https://openalex.org/W4308764898","https://openalex.org/W4308764909","https://openalex.org/W4308764932","https://openalex.org/W4308765676","https://openalex.org/W4308784155","https://openalex.org/W4308789076","https://openalex.org/W4309617474","https://openalex.org/W4309618080","https://openalex.org/W4309618420","https://openalex.org/W4309618422","https://openalex.org/W4309618424","https://openalex.org/W4309618884","https://openalex.org/W4309618902","https://openalex.org/W4309619003","https://openalex.org/W4309619177","https://openalex.org/W4317464833","https://openalex.org/W4319453786","https://openalex.org/W4320838518","https://openalex.org/W4323572060","https://openalex.org/W4365998993","https://openalex.org/W4365999110","https://openalex.org/W4365999119","https://openalex.org/W4365999247","https://openalex.org/W4366004157","https://openalex.org/W4366547294","https://openalex.org/W4366547579","https://openalex.org/W4366548675","https://openalex.org/W4366548975","https://openalex.org/W4366549723","https://openalex.org/W4366549849","https://openalex.org/W4366588458","https://openalex.org/W4366590211","https://openalex.org/W4366729216","https://openalex.org/W4375959287","https://openalex.org/W4376167053","https://openalex.org/W4376320677","https://openalex.org/W4376988619","https://openalex.org/W4377157114","https://openalex.org/W4378470321","https://openalex.org/W4379538504","https://openalex.org/W4380318991","https://openalex.org/W4380319026","https://openalex.org/W4380319170","https://openalex.org/W4380320234","https://openalex.org/W4380320333","https://openalex.org/W4380357773","https://openalex.org/W4380365635","https://openalex.org/W4382318396","https://openalex.org/W4385003447","https://openalex.org/W4385565373","https://openalex.org/W4385569698","https://openalex.org/W4385570549","https://openalex.org/W4385570984","https://openalex.org/W4385571140","https://openalex.org/W4385571263","https://openalex.org/W4385571330","https://openalex.org/W4385572232","https://openalex.org/W4386044339","https://openalex.org/W4386242308","https://openalex.org/W4386242338","https://openalex.org/W4386249192","https://openalex.org/W4387329169","https://openalex.org/W4387344795","https://openalex.org/W4387344927","https://openalex.org/W4387344951","https://openalex.org/W4387345085","https://openalex.org/W4388161877","https://openalex.org/W4388163065","https://openalex.org/W4392120502","https://openalex.org/W4392223038","https://openalex.org/W4392929788","https://openalex.org/W4395960675","https://openalex.org/W4396218396","https://openalex.org/W4396218868","https://openalex.org/W4396218892","https://openalex.org/W4396230950","https://openalex.org/W4396232207","https://openalex.org/W4396243732","https://openalex.org/W4396786982","https://openalex.org/W4396788545","https://openalex.org/W4396827246","https://openalex.org/W4396831993","https://openalex.org/W4396832030","https://openalex.org/W4396832612","https://openalex.org/W4396833070","https://openalex.org/W4396833096","https://openalex.org/W4399353342","https://openalex.org/W4399353366","https://openalex.org/W4399362445","https://openalex.org/W4399363276","https://openalex.org/W4399363341","https://openalex.org/W4399363436","https://openalex.org/W4399363470","https://openalex.org/W4399364539","https://openalex.org/W4399365301","https://openalex.org/W4400949264","https://openalex.org/W4401042630","https://openalex.org/W4401381287","https://openalex.org/W4402667058","https://openalex.org/W4402670982","https://openalex.org/W4402671145","https://openalex.org/W4402671530","https://openalex.org/W4402671543","https://openalex.org/W4402671572","https://openalex.org/W4402671698","https://openalex.org/W4402671727","https://openalex.org/W4402671803","https://openalex.org/W4402671809","https://openalex.org/W4402671916","https://openalex.org/W4402683939","https://openalex.org/W4403444997","https://openalex.org/W4403680687","https://openalex.org/W4403681490","https://openalex.org/W4403681535","https://openalex.org/W4403920278","https://openalex.org/W4404172412","https://openalex.org/W4404172434","https://openalex.org/W4404172469","https://openalex.org/W4404173724","https://openalex.org/W4404173826","https://openalex.org/W4407310184","https://openalex.org/W4407695830","https://openalex.org/W4407771402","https://openalex.org/W4407806269","https://openalex.org/W4409696766","https://openalex.org/W4409736028","https://openalex.org/W4409736107","https://openalex.org/W4409736126","https://openalex.org/W4409736307","https://openalex.org/W4409746085","https://openalex.org/W4409746205","https://openalex.org/W4409746924","https://openalex.org/W4409747817","https://openalex.org/W4409748981","https://openalex.org/W4409748992","https://openalex.org/W4409749188","https://openalex.org/W4409750046","https://openalex.org/W4409767054","https://openalex.org/W4409884130","https://openalex.org/W4409884264","https://openalex.org/W4410049598","https://openalex.org/W4410061389","https://openalex.org/W4410512536","https://openalex.org/W4410512567","https://openalex.org/W4410537238","https://openalex.org/W4410538176","https://openalex.org/W4411120843","https://openalex.org/W4411541754","https://openalex.org/W4411542006","https://openalex.org/W4411542029","https://openalex.org/W4411549867","https://openalex.org/W4411550047","https://openalex.org/W4411550093","https://openalex.org/W4411550181","https://openalex.org/W4411550383","https://openalex.org/W4411550419","https://openalex.org/W4411550425","https://openalex.org/W4411550467","https://openalex.org/W4411550496","https://openalex.org/W4411550524","https://openalex.org/W4411550526","https://openalex.org/W4411550530","https://openalex.org/W4412886735","https://openalex.org/W4412886793","https://openalex.org/W4412889634","https://openalex.org/W4412889765","https://openalex.org/W4412889798","https://openalex.org/W4412889819","https://openalex.org/W4412889963","https://openalex.org/W4412901119","https://openalex.org/W4412944751","https://openalex.org/W4412944813","https://openalex.org/W4412944862","https://openalex.org/W4412944944","https://openalex.org/W4412944953","https://openalex.org/W4412944960","https://openalex.org/W4412944984","https://openalex.org/W4412945174","https://openalex.org/W4412945178","https://openalex.org/W4412945190","https://openalex.org/W4412945360","https://openalex.org/W4412945425","https://openalex.org/W4412945467","https://openalex.org/W4412945629","https://openalex.org/W4415256708","https://openalex.org/W4415256895","https://openalex.org/W4415257586","https://openalex.org/W4415259397","https://openalex.org/W4415654100","https://openalex.org/W4416177322"],"related_works":[],"abstract_inverted_index":{"In":[0],"machine":[1],"learning,":[2],"\u201cground":[3,20],"truth\u201d":[4,21],"refers":[5],"to":[6,12,146,200],"the":[7,18,65,92,100,115,132,151,192,202],"assumed":[8],"correct":[9],"labels":[10],"used":[11],"train":[13],"and":[14,50,62,109],"evaluate":[15],"models.":[16,176],"However,":[17],"foundational":[19],"paradigm":[22],"rests":[23],"on":[24,102],"a":[25,37,168,184,196],"positivistic":[26],"fallacy":[27],"that":[28,71,84,190],"treats":[29],"human":[30,112,205],"disagreement":[31,166],"as":[32,126,160,167],"technical":[33],"noise":[34],"rather":[35],"than":[36],"vital":[38],"sociotechnical":[39],"signal.":[40],"This":[41],"systematic":[42],"literature":[43],"review":[44],"analyzes":[45],"research":[46],"published":[47],"between":[48],"2020":[49],"2025":[51],"across":[52],"seven":[53],"premier":[54],"venues:":[55],"ACL,":[56],"AIES,":[57],"CHI,":[58],"CSCW,":[59],"EAAMO,":[60],"FAccT,":[61],"NeurIPS,":[63],"investigating":[64],"mechanisms":[66],"in":[67,87],"data":[68,137],"annotation":[69,188],"practices":[70],"facilitate":[72],"this":[73],"\u201cconsensus":[74],"trap\u201d.":[75],"Our":[76],"reflexive":[77],"thematic":[78],"analysis":[79],"of":[80,135,204],"346":[81],"papers":[82],"reveals":[83],"systemic":[85,180],"failures":[86],"positional":[88],"legibility,":[89],"combined":[90],"with":[91],"recent":[93],"architectural":[94],"shift":[95,191],"toward":[96],"human-as-verifier":[97],"models,":[98],"specifically":[99],"reliance":[101],"model-mediated":[103],"annotations,":[104],"introduce":[105],"deep-seated":[106],"anchoring":[107],"bias":[108],"effectively":[110],"remove":[111],"voices":[113],"from":[114,194],"loop.":[116],"We":[117],"further":[118],"demonstrate":[119],"how":[120],"geographic":[121],"hegemony":[122],"imposes":[123],"Western":[124],"norms":[125],"universal":[127],"benchmarks,":[128],"often":[129],"enforced":[130],"by":[131],"performative":[133],"alignment":[134],"precarious":[136],"workers":[138],"who":[139],"prioritize":[140],"requester":[141],"compliance":[142],"over":[143],"honest":[144],"subjectivity":[145],"avoid":[147],"economic":[148],"penalties.":[149],"Critiquing":[150],"\u201cnoisy":[152],"sensor\u201d":[153],"fallacy,":[154],"where":[155],"statistical":[156],"models":[157],"misdiagnose":[158],"pluralism":[159],"error,":[161],"we":[162,182],"argue":[163],"for":[164,172,186],"reclaiming":[165],"high-fidelity":[169],"signal":[170],"essential":[171],"building":[173],"culturally":[174],"competent":[175],"To":[177],"address":[178],"these":[179],"tensions,":[181],"propose":[183],"roadmap":[185],"pluralistic":[187],"infrastructures":[189],"objective":[193],"discovering":[195],"singular":[197],"\u201cright\u201d":[198],"answer":[199],"mapping":[201],"diversity":[203],"experience.":[206]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-06-24T00:00:00"}