{"id":"https://openalex.org/W2952929029","doi":"https://doi.org/10.18653/v1/p19-1162","title":"A Transparent Framework for Evaluating Unintended Demographic Bias in Word Embeddings","display_name":"A Transparent Framework for Evaluating Unintended Demographic Bias in Word Embeddings","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2952929029","doi":"https://doi.org/10.18653/v1/p19-1162","mag":"2952929029"},"language":"en","primary_location":{"id":"doi:10.18653/v1/p19-1162","doi":"https://doi.org/10.18653/v1/p19-1162","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1162","pdf_url":null,"source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.18653/v1/p19-1162","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Chris Sweeney","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chris Sweeney","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5112035956","display_name":"Maryam Najafian","orcid":"https://orcid.org/0009-0005-0438-7737","observed_orcids":["https://orcid.org/0009-0005-0438-7737"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maryam Najafian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":9.3449,"has_fulltext":false,"cited_by_count":59,"citation_normalized_percentile":{"value":0.97494653,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1662","last_page":"1667"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9696999788284302,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9696999788284302,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.00800000037997961,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.006899999920278788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/word-embeddings","display_name":"word embeddings","score":0.9570000171661377},{"id":"https://openalex.org/keywords/demographic-bias","display_name":"demographic bias","score":0.9190000295639038},{"id":"https://openalex.org/keywords/fairness-evaluation","display_name":"fairness evaluation","score":0.75},{"id":"https://openalex.org/keywords/protected-groups","display_name":"protected groups","score":0.8960000276565552},{"id":"https://openalex.org/keywords/discrimination","display_name":"discrimination","score":0.460999995470047},{"id":"https://openalex.org/keywords/sentiment-analysis","display_name":"sentiment analysis","score":0.4169999957084656},{"id":"https://openalex.org/keywords/bias-metrics","display_name":"bias metrics","score":0.3970000147819519}],"concepts":[{"id":"https://openalex.org/C2777462759","wikidata":"https://www.wikidata.org/wiki/Q18395344","display_name":"Word embedding","level":3,"score":0.7159879207611084},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.702045738697052},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.67140132188797},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6541125178337097},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6089027523994446},{"id":"https://openalex.org/C2776889888","wikidata":"https://www.wikidata.org/wiki/Q1135789","display_name":"Unintended consequences","level":2,"score":0.589505672454834},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5652071237564087},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4884580671787262},{"id":"https://openalex.org/C66402592","wikidata":"https://www.wikidata.org/wiki/Q2271421","display_name":"Sentiment analysis","level":2,"score":0.45741552114486694},{"id":"https://openalex.org/C198043062","wikidata":"https://www.wikidata.org/wiki/Q180953","display_name":"Metric space","level":2,"score":0.4388037621974945},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.42373746633529663},{"id":"https://openalex.org/C2780876879","wikidata":"https://www.wikidata.org/wiki/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.42098402976989746},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.2275623381137848},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.22571614384651184},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.13384869694709778},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.07826787233352661},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/p19-1162","doi":"https://doi.org/10.18653/v1/p19-1162","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1162","pdf_url":null,"source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/p19-1162","doi":"https://doi.org/10.18653/v1/p19-1162","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1162","pdf_url":null,"source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities","score":0.7578999996185303}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":21,"referenced_works":["https://openalex.org/W2022204871","https://openalex.org/W2108646579","https://openalex.org/W2145451908","https://openalex.org/W2153579005","https://openalex.org/W2160660844","https://openalex.org/W2250539671","https://openalex.org/W2251738400","https://openalex.org/W2462418454","https://openalex.org/W2483215953","https://openalex.org/W2510955516","https://openalex.org/W2511234952","https://openalex.org/W2549762342","https://openalex.org/W2607719644","https://openalex.org/W2728567418","https://openalex.org/W2769358515","https://openalex.org/W2784397426","https://openalex.org/W2787420813","https://openalex.org/W2787481916","https://openalex.org/W2802105481","https://openalex.org/W2962787423","https://openalex.org/W2964207259"],"related_works":["https://openalex.org/W2998368506","https://openalex.org/W4287887636","https://openalex.org/W3172284381","https://openalex.org/W3035448343","https://openalex.org/W3016719704","https://openalex.org/W4402615953","https://openalex.org/W3013854004","https://openalex.org/W3184453204","https://openalex.org/W3212097072","https://openalex.org/W2959360485"],"abstract_inverted_index":{"Word":[0,36],"embedding":[1,53,95],"models":[2],"have":[3],"gained":[4],"a":[5,79],"lot":[6],"of":[7],"traction":[8],"in":[9,51,70,106,136],"the":[10,35,52,65,110,134],"Natural":[11],"Language":[12],"Processing":[13],"community,":[14],"however,":[15],"they":[16,56],"suffer":[17],"from":[18,119],"unintended":[19,49],"demographic":[20,116],"biases.":[21],"Most":[22],"approaches":[23,43],"to":[24,58,92],"evaluate":[25],"these":[26,42],"biases":[27,50],"rely":[28],"on":[29],"vector":[30,54],"space":[31],"based":[32],"metrics":[33],"like":[34],"Embedding":[37],"Association":[38],"Test":[39],"(WEAT).":[40],"While":[41],"offer":[44,59],"great":[45],"geometric":[46],"insights":[47],"into":[48,133],"space,":[55],"fail":[57],"an":[60],"interpretable":[61],"meaning":[62],"for":[63,84],"how":[64],"embeddings":[66,108],"could":[67],"cause":[68],"discrimination":[69,86],"downstream":[71],"NLP":[72],"applications.":[73],"In":[74],"this":[75],"work,":[76],"we":[77],"present":[78],"transparent":[80],"framework":[81,127],"and":[82,128],"metric":[83,98,129],"evaluating":[85],"across":[87],"protected":[88,121],"groups":[89],"with":[90,115],"respect":[91],"their":[93],"word":[94,107,137],"bias.":[96],"Our":[97],"(Relative":[99],"Negative":[100],"Sentiment":[101],"Bias,":[102],"RNSB)":[103],"measures":[104],"fairness":[105],"via":[109],"relative":[111],"negative":[112],"sentiment":[113],"associated":[114],"identity":[117],"terms":[118],"various":[120],"groups.":[122],"We":[123],"show":[124],"that":[125],"our":[126],"enable":[130],"useful":[131],"analysis":[132],"bias":[135],"embeddings.":[138]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":15},{"year":2022,"cited_by_count":12},{"year":2021,"cited_by_count":13},{"year":2020,"cited_by_count":11}],"updated_date":"2026-11-03T10:50:19.383937","created_date":"2019-06-27T00:00:00"}