{"id":"https://openalex.org/W3206428286","doi":"https://doi.org/10.1162/tacl_a_00449","title":"Dealing with Disagreements: Looking Beyond the Majority Vote in Subjective Annotations","display_name":"Dealing with Disagreements: Looking Beyond the Majority Vote in Subjective Annotations","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W3206428286","doi":"https://doi.org/10.1162/tacl_a_00449","mag":"3206428286"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00449","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00449","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00449/1986597/tacl_a_00449.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00449/1986597/tacl_a_00449.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000967986","display_name":"Aida Mostafazadeh Davani","orcid":"https://orcid.org/0000-0002-7013-1810"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Aida Mostafazadeh Davani","raw_affiliation_strings":["University of Southern California, USA. mostafaz@usc.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Southern California, USA. mostafaz@usc.edu","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012652561","display_name":"Mark D\u00edaz","orcid":"https://orcid.org/0000-0003-0167-9839"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Mark D\u00edaz","raw_affiliation_strings":["Google Research, USA. markdiaz@google.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research, USA. markdiaz@google.com","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5019297976","display_name":"Vinodkumar Prabhakaran","orcid":"https://orcid.org/0000-0003-3329-2305"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Vinodkumar Prabhakaran","raw_affiliation_strings":["Google Research, USA. vinodkpg@google.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research, USA. vinodkpg@google.com","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5000967986","https://openalex.org/A5012652561","https://openalex.org/A5019297976"],"corresponding_institution_ids":["https://openalex.org/I1174212","https://openalex.org/I1291425158"],"apc_list":null,"apc_paid":null,"fwci":20.4384,"has_fulltext":false,"cited_by_count":209,"citation_normalized_percentile":{"value":0.99563184,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":100},"biblio":{"volume":"10","issue":null,"first_page":"92","last_page":"110"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.9958999752998352,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9873999953269958,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8678797483444214},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6882572770118713},{"id":"https://openalex.org/keywords/voting","display_name":"Voting","score":0.6464080810546875},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.6038460731506348},{"id":"https://openalex.org/keywords/majority-rule","display_name":"Majority rule","score":0.5925784111022949},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.5728578567504883},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5527790188789368},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5226418972015381},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.471820205450058},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4708896279335022}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8678797483444214},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6882572770118713},{"id":"https://openalex.org/C520049643","wikidata":"https://www.wikidata.org/wiki/Q189760","display_name":"Voting","level":3,"score":0.6464080810546875},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.6038460731506348},{"id":"https://openalex.org/C153668964","wikidata":"https://www.wikidata.org/wiki/Q27636","display_name":"Majority rule","level":2,"score":0.5925784111022949},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.5728578567504883},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5527790188789368},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5226418972015381},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.471820205450058},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4708896279335022},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1162/tacl_a_00449","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00449","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00449/1986597/tacl_a_00449.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:d53ffa307ef1449b8c10c27a9b28041c","is_oa":true,"landing_page_url":"https://doaj.org/article/d53ffa307ef1449b8c10c27a9b28041c","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 10 (2023)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00449","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00449","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00449/1986597/tacl_a_00449.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.6399999856948853,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":109,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W78136081","https://openalex.org/W80497942","https://openalex.org/W172170795","https://openalex.org/W583714418","https://openalex.org/W1513398909","https://openalex.org/W1607035479","https://openalex.org/W1663984431","https://openalex.org/W1702329404","https://openalex.org/W1799078166","https://openalex.org/W1966797434","https://openalex.org/W1970381522","https://openalex.org/W1987545044","https://openalex.org/W2003497265","https://openalex.org/W2013378101","https://openalex.org/W2050730017","https://openalex.org/W2054953618","https://openalex.org/W2066064791","https://openalex.org/W2075718943","https://openalex.org/W2105468141","https://openalex.org/W2114269021","https://openalex.org/W2114524997","https://openalex.org/W2126581182","https://openalex.org/W2133341045","https://openalex.org/W2134031328","https://openalex.org/W2147638277","https://openalex.org/W2170876097","https://openalex.org/W2208202786","https://openalex.org/W2250493512","https://openalex.org/W2251311344","https://openalex.org/W2251551120","https://openalex.org/W2251818274","https://openalex.org/W2251892406","https://openalex.org/W2290009368","https://openalex.org/W2295951612","https://openalex.org/W2405202646","https://openalex.org/W2473555522","https://openalex.org/W2531327146","https://openalex.org/W2552810951","https://openalex.org/W2563826943","https://openalex.org/W2595653137","https://openalex.org/W2740168486","https://openalex.org/W2741036097","https://openalex.org/W2759083144","https://openalex.org/W2791170418","https://openalex.org/W2795975316","https://openalex.org/W2799915114","https://openalex.org/W2801959488","https://openalex.org/W2809006044","https://openalex.org/W2888507760","https://openalex.org/W2899689163","https://openalex.org/W2901114707","https://openalex.org/W2926526813","https://openalex.org/W2937151161","https://openalex.org/W2949678053","https://openalex.org/W2959143276","https://openalex.org/W2963052389","https://openalex.org/W2963341956","https://openalex.org/W2963854351","https://openalex.org/W2963873807","https://openalex.org/W2963943967","https://openalex.org/W2964059111","https://openalex.org/W2964121744","https://openalex.org/W2966939796","https://openalex.org/W2970395295","https://openalex.org/W2970442950","https://openalex.org/W2970670424","https://openalex.org/W2972735048","https://openalex.org/W2979826702","https://openalex.org/W2988207158","https://openalex.org/W2989344603","https://openalex.org/W2989817717","https://openalex.org/W2993013587","https://openalex.org/W3011385529","https://openalex.org/W3012395103","https://openalex.org/W3034323190","https://openalex.org/W3035296331","https://openalex.org/W3046403583","https://openalex.org/W3093338286","https://openalex.org/W3098173943","https://openalex.org/W3100363073","https://openalex.org/W3103992175","https://openalex.org/W3106219395","https://openalex.org/W3121735664","https://openalex.org/W3155742828","https://openalex.org/W3161695571","https://openalex.org/W3163078977","https://openalex.org/W3166716308","https://openalex.org/W3176580738","https://openalex.org/W3214105842","https://openalex.org/W4233368769","https://openalex.org/W4301100068","https://openalex.org/W4310895557","https://openalex.org/W6603174692","https://openalex.org/W6606988585","https://openalex.org/W6617145748","https://openalex.org/W6631190155","https://openalex.org/W6638003574","https://openalex.org/W6678758578","https://openalex.org/W6678923525","https://openalex.org/W6688022499","https://openalex.org/W6691258510","https://openalex.org/W6691296735","https://openalex.org/W6691776783","https://openalex.org/W6728622933","https://openalex.org/W6755207826","https://openalex.org/W6766738310","https://openalex.org/W6767608665","https://openalex.org/W6781215487"],"related_works":["https://openalex.org/W2361861616","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W3121841074","https://openalex.org/W2901018557","https://openalex.org/W2055572829","https://openalex.org/W1894159578","https://openalex.org/W4297796115","https://openalex.org/W3125086856","https://openalex.org/W2295086410"],"abstract_inverted_index":{"Abstract":[0],"Majority":[1],"voting":[2],"and":[3,13,35,49],"averaging":[4],"are":[5,62],"common":[6,102],"approaches":[7],"used":[8],"to":[9,68,75,125,139,157,170],"resolve":[10],"annotator":[11],"disagreements":[12,53,151],"derive":[14],"single":[15,70],"ground":[16,71],"truth":[17],"labels":[18,120],"from":[19],"multiple":[20],"annotations.":[21],"However,":[22],"annotators":[23],"may":[24,54],"systematically":[25],"disagree":[26],"with":[27,149],"one":[28],"another,":[29],"often":[30,63],"reflecting":[31],"their":[32],"individual":[33],"biases":[34],"values,":[36],"especially":[37,161],"in":[38,58,121,142,163],"the":[39,80,106,122],"case":[40],"of":[41,82,105],"subjective":[42],"tasks":[43,60],"such":[44,59],"as":[45,96],"detecting":[46],"affect,":[47],"aggression,":[48],"hate":[50],"speech.":[51],"Annotator":[52],"capture":[55],"important":[56],"nuances":[57],"that":[61,110],"ignored":[64],"while":[65,99],"aggregating":[66,119],"annotations":[67],"a":[69,101,137,172],"truth.":[72],"In":[73,85],"order":[74],"address":[76],"this,":[77],"we":[78,145],"investigate":[79],"efficacy":[81],"multi-annotator":[83],"models.":[84],"particular,":[86],"our":[87],"multi-task":[88],"based":[89],"approach":[90,112,134],"treats":[91],"predicting":[92],"each":[93],"annotators\u2019":[94],"judgements":[95],"separate":[97],"subtasks,":[98],"sharing":[100],"learned":[103],"representation":[104],"task.":[107],"We":[108],"show":[109],"this":[111],"yields":[113],"same":[114],"or":[115],"better":[116,147],"performance":[117],"than":[118,152],"data":[123],"prior":[124],"training":[126],"across":[127],"seven":[128],"different":[129],"binary":[130],"classification":[131],"tasks.":[132],"Our":[133],"also":[135],"provides":[136],"way":[138],"estimate":[140],"uncertainty":[141,159],"predictions,":[143],"which":[144],"demonstrate":[146],"correlate":[148],"annotation":[150],"traditional":[153],"methods.":[154],"Being":[155],"able":[156],"model":[158],"is":[160,174],"useful":[162],"deployment":[164],"scenarios":[165],"where":[166],"knowing":[167],"when":[168],"not":[169],"make":[171],"prediction":[173],"important.":[175]},"counts_by_year":[{"year":2026,"cited_by_count":31},{"year":2025,"cited_by_count":43},{"year":2024,"cited_by_count":37},{"year":2023,"cited_by_count":65},{"year":2022,"cited_by_count":31},{"year":2021,"cited_by_count":2}],"updated_date":"2026-07-31T08:31:51.225901","created_date":"2025-10-10T00:00:00"}