{"id":"https://openalex.org/W2972539067","doi":"https://doi.org/10.18653/v1/d19-1578","title":"Perturbation Sensitivity Analysis to Detect Unintended Model Biases","display_name":"Perturbation Sensitivity Analysis to Detect Unintended Model Biases","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2972539067","doi":"https://doi.org/10.18653/v1/d19-1578","mag":"2972539067"},"language":"en","primary_location":{"id":"doi:10.18653/v1/d19-1578","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d19-1578","pdf_url":"https://www.aclweb.org/anthology/D19-1578.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/D19-1578.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5019297976","display_name":"Vinodkumar Prabhakaran","orcid":"https://orcid.org/0000-0003-3329-2305"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Vinodkumar Prabhakaran","raw_affiliation_strings":["Google Brain San Francisco, CA, USA","Google,,,,,"],"affiliations":[{"raw_affiliation_string":"Google Brain San Francisco, CA, USA","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071599724","display_name":"Ben Hutchinson","orcid":"https://orcid.org/0000-0003-2253-6204"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ben Hutchinson","raw_affiliation_strings":["Google Brain San Francisco, CA, USA","Google,,,,,"],"affiliations":[{"raw_affiliation_string":"Google Brain San Francisco, CA, USA","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046235098","display_name":"Margaret Mitchell","orcid":"https://orcid.org/0000-0001-7043-6545"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Margaret Mitchell","raw_affiliation_strings":["Google Brain Seattle, WA, USA","Google,,,,,"],"affiliations":[{"raw_affiliation_string":"Google Brain Seattle, WA, USA","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5019297976"],"corresponding_institution_ids":["https://openalex.org/I1291425158"],"apc_list":null,"apc_paid":null,"fwci":1.22894196,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.83990384,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"5739","last_page":"5744"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7619315385818481},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.7017741203308105},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6979345083236694},{"id":"https://openalex.org/keywords/phrase","display_name":"Phrase","score":0.6938605904579163},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6617638468742371},{"id":"https://openalex.org/keywords/sentiment-analysis","display_name":"Sentiment analysis","score":0.6192198991775513},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5947616100311279},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.5134554505348206}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7619315385818481},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.7017741203308105},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6979345083236694},{"id":"https://openalex.org/C2776224158","wikidata":"https://www.wikidata.org/wiki/Q187931","display_name":"Phrase","level":2,"score":0.6938605904579163},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6617638468742371},{"id":"https://openalex.org/C66402592","wikidata":"https://www.wikidata.org/wiki/Q2271421","display_name":"Sentiment analysis","level":2,"score":0.6192198991775513},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5947616100311279},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.5134554505348206}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.18653/v1/d19-1578","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d19-1578","pdf_url":"https://www.aclweb.org/anthology/D19-1578.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1910.04210","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1910.04210","pdf_url":"https://arxiv.org/pdf/1910.04210","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"mag:2972539067","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1910.04210","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1910.04210","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1910.04210","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.18653/v1/d19-1578","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d19-1578","pdf_url":"https://www.aclweb.org/anthology/D19-1578.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7099999785423279}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2972539067.pdf","grobid_xml":"https://content.openalex.org/works/W2972539067.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W2014352947","https://openalex.org/W2100960835","https://openalex.org/W2483215953","https://openalex.org/W2562979205","https://openalex.org/W2728567418","https://openalex.org/W2791170418","https://openalex.org/W2802105481","https://openalex.org/W2805572053","https://openalex.org/W2893425640","https://openalex.org/W2897154134","https://openalex.org/W2897978524","https://openalex.org/W2900885930","https://openalex.org/W2920114910","https://openalex.org/W2920807444","https://openalex.org/W2954275542","https://openalex.org/W2962833164","https://openalex.org/W2963078909","https://openalex.org/W3037697022"],"related_works":["https://openalex.org/W2989344603","https://openalex.org/W3101767999","https://openalex.org/W147145573","https://openalex.org/W2794809337","https://openalex.org/W2134175060","https://openalex.org/W2805817105","https://openalex.org/W3003801295","https://openalex.org/W3020968600","https://openalex.org/W2888670907","https://openalex.org/W2117088648","https://openalex.org/W3211989786","https://openalex.org/W2056695536","https://openalex.org/W2197913429","https://openalex.org/W3035199167","https://openalex.org/W3205874557","https://openalex.org/W24083617","https://openalex.org/W2741642835","https://openalex.org/W2463188703","https://openalex.org/W1506522508","https://openalex.org/W2784778675"],"abstract_inverted_index":{"Data-driven":[0],"statistical":[1],"Natural":[2],"Language":[3],"Processing":[4],"(NLP)":[5],"techniques":[6],"leverage":[7],"large":[8],"amounts":[9],"of":[10,79,82,155],"language":[11,22,181],"data":[12,23,32],"to":[13,41,56,64,141],"build":[14],"models":[15,38,165],"that":[16,76],"can":[17],"understand":[18],"language.":[19],"However,":[20],"most":[21],"reflect":[24],"the":[25,29,31,80,114,153],"public":[26],"discourse":[27],"at":[28,48],"time":[30],"was":[33],"produced,":[34],"and":[35,70,88,144,170],"hence":[36],"NLP":[37,61,164],"are":[39,77],"susceptible":[40],"learning":[42],"incidental":[43],"associations":[44],"around":[45],"named":[46,142],"referents":[47],"a":[49,95,101,128,167,171],"particular":[50],"point":[51],"in":[52,54,86,94,179],"time,":[53],"addition":[55],"general":[57,96],"linguistic":[58],"meaning.":[59],"An":[60],"system":[62],"designed":[63],"model":[65,138,169,173],"notions":[66],"such":[67,83,103],"as":[68,104,112,117],"sentiment":[69,98,116,168],"toxicity":[71,172],"should":[72,109],"ideally":[73],"produce":[74],"scores":[75],"independent":[78],"identity":[81],"entities":[84],"mentioned":[85],"text":[87],"their":[89],"social":[90],"associations.":[91],"For":[92],"example,":[93],"purpose":[97],"analysis":[99,157],"system,":[100],"phrase":[102],"I":[105,118],"hate":[106,119],"Katy":[107],"Perry":[108],"be":[110],"interpreted":[111],"having":[113],"same":[115],"Taylor":[120],"Swift.":[121],"Based":[122],"on":[123,161,176],"this":[124,156],"idea,":[125],"we":[126],"propose":[127],"generic":[129],"evaluation":[130],"framework,":[131],"Perturbation":[132],"Sensitivity":[133],"Analysis,":[134],"which":[135],"detects":[136],"unintended":[137],"biases":[139],"related":[140],"entities,":[143],"requires":[145],"no":[146],"new":[147],"annotations":[148],"or":[149],"corpora.":[150],"We":[151],"demonstrate":[152],"utility":[154],"by":[158],"employing":[159],"it":[160],"two":[162],"different":[163,184],"---":[166,174],"applied":[175],"online":[177],"comments":[178],"English":[180],"from":[182],"four":[183],"genres.":[185]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":3}],"updated_date":"2026-02-09T09:26:11.010843","created_date":"2025-10-10T00:00:00"}