{"id":"https://openalex.org/W2949678053","doi":"https://doi.org/10.18653/v1/p19-1163","title":"The Risk of Racial Bias in Hate Speech Detection","display_name":"The Risk of Racial Bias in Hate Speech Detection","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2949678053","doi":"https://doi.org/10.18653/v1/p19-1163","mag":"2949678053"},"language":"en","primary_location":{"id":"doi:10.18653/v1/p19-1163","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1163","pdf_url":"https://www.aclweb.org/anthology/P19-1163.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/P19-1163.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015128745","display_name":"Maarten Sap","orcid":"https://orcid.org/0000-0002-0701-4654"},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]},{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Maarten Sap","raw_affiliation_strings":["School of Computer Science & Engineering, University of Washington, Seattle, USA"],"affiliations":[{"raw_affiliation_string":"School of Computer Science & Engineering, University of Washington, Seattle, USA","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070767948","display_name":"Dallas Card","orcid":"https://orcid.org/0000-0001-5573-8836"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dallas Card","raw_affiliation_strings":["Machine Learning Department, Carnegie Mellon University, Pittsburgh, USA"],"affiliations":[{"raw_affiliation_string":"Machine Learning Department, Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015792268","display_name":"Saadia Gabriel","orcid":"https://orcid.org/0009-0001-9353-951X"},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]},{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saadia Gabriel","raw_affiliation_strings":["School of Computer Science & Engineering, University of Washington, Seattle, USA"],"affiliations":[{"raw_affiliation_string":"School of Computer Science & Engineering, University of Washington, Seattle, USA","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102992157","display_name":"Yejin Choi","orcid":"https://orcid.org/0000-0003-3032-5378"},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yejin Choi","raw_affiliation_strings":["Allen Institute for Artificial Intelligence, Seattle, USA"],"affiliations":[{"raw_affiliation_string":"Allen Institute for Artificial Intelligence, Seattle, USA","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5088517824","display_name":"Noah A. Smith","orcid":"https://orcid.org/0000-0002-2310-6380"},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Noah A. Smith","raw_affiliation_strings":["Allen Institute for Artificial Intelligence, Seattle, USA"],"affiliations":[{"raw_affiliation_string":"Allen Institute for Artificial Intelligence, Seattle, USA","institution_ids":["https://openalex.org/I4210156221"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5015128745"],"corresponding_institution_ids":["https://openalex.org/I201448701","https://openalex.org/I58610484"],"apc_list":null,"apc_paid":null,"fwci":63.7297,"has_fulltext":true,"cited_by_count":756,"citation_normalized_percentile":{"value":0.99908047,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1668","last_page":"1678"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10216","display_name":"Migration, Refugees, and Integration","score":0.9469000101089478,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11397","display_name":"Populism, Right-Wing Movements","score":0.944599986076355,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/offensive","display_name":"Offensive","score":0.9265945553779602},{"id":"https://openalex.org/keywords/harm","display_name":"Harm","score":0.7266355156898499},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6231341361999512},{"id":"https://openalex.org/keywords/priming","display_name":"Priming (agriculture)","score":0.5377552509307861},{"id":"https://openalex.org/keywords/african-american","display_name":"African american","score":0.5112097263336182},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.47920259833335876},{"id":"https://openalex.org/keywords/racial-bias","display_name":"Racial bias","score":0.4769771993160248},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.44899624586105347},{"id":"https://openalex.org/keywords/racism","display_name":"Racism","score":0.4183374047279358},{"id":"https://openalex.org/keywords/voice-activity-detection","display_name":"Voice activity detection","score":0.4173860549926758},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.41169652342796326},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.40939027070999146},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.32173532247543335},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.22095119953155518},{"id":"https://openalex.org/keywords/speech-processing","display_name":"Speech processing","score":0.20423853397369385},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.13526231050491333},{"id":"https://openalex.org/keywords/history","display_name":"History","score":0.11769846081733704},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.07445791363716125}],"concepts":[{"id":"https://openalex.org/C176856949","wikidata":"https://www.wikidata.org/wiki/Q2001676","display_name":"Offensive","level":2,"score":0.9265945553779602},{"id":"https://openalex.org/C2777363581","wikidata":"https://www.wikidata.org/wiki/Q15098235","display_name":"Harm","level":2,"score":0.7266355156898499},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6231341361999512},{"id":"https://openalex.org/C81444415","wikidata":"https://www.wikidata.org/wiki/Q7243535","display_name":"Priming (agriculture)","level":3,"score":0.5377552509307861},{"id":"https://openalex.org/C2987028688","wikidata":"https://www.wikidata.org/wiki/Q49085","display_name":"African american","level":2,"score":0.5112097263336182},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.47920259833335876},{"id":"https://openalex.org/C2992700788","wikidata":"https://www.wikidata.org/wiki/Q8461","display_name":"Racial bias","level":3,"score":0.4769771993160248},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.44899624586105347},{"id":"https://openalex.org/C139838865","wikidata":"https://www.wikidata.org/wiki/Q8461","display_name":"Racism","level":2,"score":0.4183374047279358},{"id":"https://openalex.org/C204201278","wikidata":"https://www.wikidata.org/wiki/Q1332614","display_name":"Voice activity detection","level":3,"score":0.4173860549926758},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41169652342796326},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.40939027070999146},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.32173532247543335},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.22095119953155518},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.20423853397369385},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.13526231050491333},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.11769846081733704},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07445791363716125},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C2549261","wikidata":"https://www.wikidata.org/wiki/Q43455","display_name":"Ethnology","level":1,"score":0.0},{"id":"https://openalex.org/C100701293","wikidata":"https://www.wikidata.org/wiki/Q193838","display_name":"Germination","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/p19-1163","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1163","pdf_url":"https://www.aclweb.org/anthology/P19-1163.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/p19-1163","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1163","pdf_url":"https://www.aclweb.org/anthology/P19-1163.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.5400000214576721,"display_name":"Reduced inequalities"}],"awards":[{"id":"https://openalex.org/G5551313627","display_name":"RI: Small: ConnotationNet: Modeling Non-Literal Meaning in Context","funder_award_id":"1714566","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2949678053.pdf","grobid_xml":"https://content.openalex.org/works/W2949678053.grobid-xml"},"referenced_works_count":42,"referenced_works":["https://openalex.org/W80056832","https://openalex.org/W1871142974","https://openalex.org/W1978599464","https://openalex.org/W2011588348","https://openalex.org/W2094862034","https://openalex.org/W2106662182","https://openalex.org/W2112205306","https://openalex.org/W2151441810","https://openalex.org/W2162068690","https://openalex.org/W2167884694","https://openalex.org/W2188158549","https://openalex.org/W2250539671","https://openalex.org/W2470673105","https://openalex.org/W2473555522","https://openalex.org/W2505776896","https://openalex.org/W2508354372","https://openalex.org/W2530395818","https://openalex.org/W2540646130","https://openalex.org/W2562607067","https://openalex.org/W2595653137","https://openalex.org/W2734862619","https://openalex.org/W2740168486","https://openalex.org/W2784778675","https://openalex.org/W2785615365","https://openalex.org/W2791170418","https://openalex.org/W2794585602","https://openalex.org/W2800170985","https://openalex.org/W2806872289","https://openalex.org/W2863494293","https://openalex.org/W2883957017","https://openalex.org/W2884692143","https://openalex.org/W2949089361","https://openalex.org/W2953692061","https://openalex.org/W2954992865","https://openalex.org/W2962990575","https://openalex.org/W2963481894","https://openalex.org/W2964235839","https://openalex.org/W2966172639","https://openalex.org/W4239773584","https://openalex.org/W4240335515","https://openalex.org/W4250532188","https://openalex.org/W4295910202"],"related_works":["https://openalex.org/W1568520348","https://openalex.org/W3214407891","https://openalex.org/W3194113117","https://openalex.org/W3213194066","https://openalex.org/W2967125893","https://openalex.org/W268355439","https://openalex.org/W4287020359","https://openalex.org/W4385323698","https://openalex.org/W2385362579","https://openalex.org/W3197406908"],"abstract_inverted_index":{"We":[0,26],"investigate":[1],"how":[2],"annotators'":[3],"insensitivity":[4],"to":[5,11,75,80,86,97,122],"differences":[6],"in":[7,14,43,102],"dialect":[8,91,116],"can":[9],"lead":[10],"racial":[12,100],"bias":[13,101],"automatic":[15],"hate":[16,46],"speech":[17,47],"detection":[18],"models,":[19],"potentially":[20],"amplifying":[21],"harm":[22],"against":[23],"minority":[24],"populations.":[25],"first":[27],"uncover":[28],"unexpected":[29],"correlations":[30],"between":[31],"surface":[32],"markers":[33],"of":[34,41,112],"African":[35,71],"American":[36],"English":[37],"(AAE)":[38],"and":[39,59,67,92],"ratings":[40],"toxicity":[42],"several":[44],"widely-used":[45],"datasets.":[48],"Then,":[49],"we":[50,89],"show":[51],"that":[52,64,105],"models":[53],"trained":[54],"on":[55],"these":[56,61],"corpora":[57],"acquire":[58],"propagate":[60],"biases,":[62],"such":[63],"AAE":[65,114],"tweets":[66,68],"by":[69],"self-identified":[70],"Americans":[72],"are":[73,108,118],"up":[74],"two":[76],"times":[77],"more":[78],"likely":[79,121],"be":[81],"labelled":[82],"as":[83,95,126],"offensive":[84],"compared":[85],"others.":[87],"Finally,":[88],"propose":[90],"race":[93],"priming":[94],"ways":[96],"reduce":[98],"the":[99,124],"annotation,":[103],"showing":[104],"when":[106],"annotators":[107],"made":[109],"explicitly":[110],"aware":[111],"an":[113],"tweet's":[115],"they":[117],"significantly":[119],"less":[120],"label":[123],"tweet":[125],"offensive.":[127]},"counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":86},{"year":2024,"cited_by_count":80},{"year":2023,"cited_by_count":136},{"year":2022,"cited_by_count":136},{"year":2021,"cited_by_count":178},{"year":2020,"cited_by_count":114},{"year":2019,"cited_by_count":12},{"year":2018,"cited_by_count":1}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}