{"id":"https://openalex.org/W3174651299","doi":"https://doi.org/10.1162/tacl_a_00425","title":"Quantifying Social Biases in NLP: A Generalization and Empirical Comparison of Extrinsic Fairness Metrics","display_name":"Quantifying Social Biases in NLP: A Generalization and Empirical Comparison of Extrinsic Fairness Metrics","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3174651299","doi":"https://doi.org/10.1162/tacl_a_00425","mag":"3174651299"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00425","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00425","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00425/1972677/tacl_a_00425.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","datacite","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00425/1972677/tacl_a_00425.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070789267","display_name":"Paula Czarnowska","orcid":null},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Paula Czarnowska","raw_affiliation_strings":["University of Cambridge, UK. pjc211@cam.ac.uk"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Cambridge, UK. pjc211@cam.ac.uk","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008851653","display_name":"Yogarshi Vyas","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yogarshi Vyas","raw_affiliation_strings":["Amazon AI, USA. yogarshi@amazon.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon AI, USA. yogarshi@amazon.com","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"last","author":{"id":null,"display_name":"Kashif Shah","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kashif Shah","raw_affiliation_strings":["Amazon AI, USA. shahkas@amazon.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon AI, USA. shahkas@amazon.com","institution_ids":["https://openalex.org/I1311688040"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.11366744,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"9","issue":null,"first_page":"1249","last_page":"1267"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9028000235557556,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9028000235557556,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.011099999770522118,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.00860000029206276,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.7191989421844482},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6854835152626038},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.6624777317047119},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5334171652793884},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.46448999643325806},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.4488058388233185},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.44074004888534546},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.40650874376296997},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.33973458409309387},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.20867109298706055},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.18115976452827454}],"concepts":[{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.7191989421844482},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6854835152626038},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.6624777317047119},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5334171652793884},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46448999643325806},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4488058388233185},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.44074004888534546},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.40650874376296997},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.33973458409309387},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.20867109298706055},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.18115976452827454},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1162/tacl_a_00425","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00425","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00425/1972677/tacl_a_00425.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2106.14574","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2106.14574","pdf_url":"https://arxiv.org/pdf/2106.14574","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:doaj.org/article:d2acc8e1928e489f8e4ff71b5ee6cc52","is_oa":true,"landing_page_url":"https://doaj.org/article/d2acc8e1928e489f8e4ff71b5ee6cc52","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 9, Pp 1249-1267 (2021)","raw_type":"article"},{"id":"doi:10.48550/arxiv.2106.14574","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2106.14574","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00425","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00425","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00425/1972677/tacl_a_00425.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":90,"referenced_works":["https://openalex.org/W1972978214","https://openalex.org/W2004763266","https://openalex.org/W2048105386","https://openalex.org/W2100960835","https://openalex.org/W2144578941","https://openalex.org/W2147880316","https://openalex.org/W2251939518","https://openalex.org/W2598476089","https://openalex.org/W2728567418","https://openalex.org/W2734639922","https://openalex.org/W2791170418","https://openalex.org/W2799159261","https://openalex.org/W2802105481","https://openalex.org/W2805744755","https://openalex.org/W2893425640","https://openalex.org/W2901107694","https://openalex.org/W2909212904","https://openalex.org/W2914097099","https://openalex.org/W2920114910","https://openalex.org/W2920807444","https://openalex.org/W2949534740","https://openalex.org/W2949678053","https://openalex.org/W2950018712","https://openalex.org/W2950888501","https://openalex.org/W2952328691","https://openalex.org/W2952638532","https://openalex.org/W2956059299","https://openalex.org/W2962833164","https://openalex.org/W2963457723","https://openalex.org/W2963526187","https://openalex.org/W2964023221","https://openalex.org/W2971015127","https://openalex.org/W2972413484","https://openalex.org/W2972423177","https://openalex.org/W2972663293","https://openalex.org/W2972697496","https://openalex.org/W2972735048","https://openalex.org/W2973192523","https://openalex.org/W2982521860","https://openalex.org/W2989344603","https://openalex.org/W2998366771","https://openalex.org/W3007065191","https://openalex.org/W3012988362","https://openalex.org/W3013854004","https://openalex.org/W3013937328","https://openalex.org/W3019416653","https://openalex.org/W3023553115","https://openalex.org/W3026016891","https://openalex.org/W3033733989","https://openalex.org/W3034458735","https://openalex.org/W3034778990","https://openalex.org/W3035296331","https://openalex.org/W3035377849","https://openalex.org/W3035507081","https://openalex.org/W3035591180","https://openalex.org/W3037831233","https://openalex.org/W3040711047","https://openalex.org/W3089376187","https://openalex.org/W3093000437","https://openalex.org/W3094372599","https://openalex.org/W3098988204","https://openalex.org/W3099635335","https://openalex.org/W3100881017","https://openalex.org/W3101004475","https://openalex.org/W3101767999","https://openalex.org/W3105882417","https://openalex.org/W3113754522","https://openalex.org/W3176477796","https://openalex.org/W3177189402","https://openalex.org/W3195725782","https://openalex.org/W4239235761","https://openalex.org/W4288617757","https://openalex.org/W6682082992","https://openalex.org/W6691459498","https://openalex.org/W6691593892","https://openalex.org/W6721933647","https://openalex.org/W6728551298","https://openalex.org/W6740303850","https://openalex.org/W6744110554","https://openalex.org/W6759056966","https://openalex.org/W6760413209","https://openalex.org/W6762584339","https://openalex.org/W6763192249","https://openalex.org/W6766396606","https://openalex.org/W6766673545","https://openalex.org/W6767327189","https://openalex.org/W6771917389","https://openalex.org/W6774575565","https://openalex.org/W6775083272","https://openalex.org/W6794866892"],"related_works":["https://openalex.org/W3028690445","https://openalex.org/W3102961637","https://openalex.org/W80018460","https://openalex.org/W2397486511","https://openalex.org/W2912536523","https://openalex.org/W2965168627","https://openalex.org/W2995122033","https://openalex.org/W3093557617","https://openalex.org/W2965334181","https://openalex.org/W2469820973","https://openalex.org/W2888883227","https://openalex.org/W3101454641","https://openalex.org/W3121389446","https://openalex.org/W3049528174","https://openalex.org/W3164790101","https://openalex.org/W2461218209","https://openalex.org/W2314485872","https://openalex.org/W3206680219","https://openalex.org/W3089591718","https://openalex.org/W1674702894"],"abstract_inverted_index":{"Abstract":[0],"Measuring":[1],"bias":[2,91],"is":[3,15],"key":[4],"for":[5,102],"better":[6],"understanding":[7],"and":[8,45,84],"addressing":[9],"unfairness":[10],"in":[11,25,52,90,99],"NLP/ML":[12],"models.":[13],"This":[14],"often":[16],"done":[17],"via":[18,97],"fairness":[19,49,66],"metrics,":[20,67],"which":[21],"quantify":[22],"the":[23,43,48,69,87],"differences":[24,44,89,98],"a":[26,30,57],"model\u2019s":[27],"behaviour":[28],"across":[29],"range":[31,59],"of":[32,60,81],"demographic":[33],"groups.":[34],"In":[35],"this":[36],"work,":[37],"we":[38,55,74],"shed":[39],"more":[40],"light":[41],"on":[42],"similarities":[46],"between":[47,71],"metrics":[50,62,83],"used":[51],"NLP.":[53],"First,":[54],"unify":[56],"broad":[58],"existing":[61,82],"under":[63],"three":[64],"generalized":[65,104],"revealing":[68],"connections":[70],"them.":[72],"Next,":[73],"carry":[75],"out":[76],"an":[77],"extensive":[78],"empirical":[79],"comparison":[80],"demonstrate":[85],"that":[86],"observed":[88],"measurement":[92],"can":[93],"be":[94],"systematically":[95],"explained":[96],"parameter":[100],"choices":[101],"our":[103],"metrics.":[105]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2021-07-05T00:00:00"}