{"id":"https://openalex.org/W4389524209","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.230","title":"ROBBIE: Robust Bias Evaluation of Large Generative Language Models","display_name":"ROBBIE: Robust Bias Evaluation of Large Generative Language Models","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4389524209","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.230"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.emnlp-main.230","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.230","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-main.230","pdf_url":"https://aclanthology.org/2023.emnlp-main.230.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.emnlp-main.230.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5060413371","display_name":"David Esiobu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Esiobu","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109301242","display_name":"Xiaoqing Tan","orcid":"https://orcid.org/0009-0003-9293-4723","observed_orcids":["https://orcid.org/0009-0003-9293-4723"]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaoqing Tan","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080012348","display_name":"Saghar Hosseini","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saghar Hosseini","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067335278","display_name":"Megan Ung","orcid":"https://orcid.org/0000-0003-3063-3207","observed_orcids":["https://orcid.org/0000-0003-3063-3207"]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Megan Ung","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100424317","display_name":"Yuchen Zhang","orcid":"https://orcid.org/0000-0001-8536-7175","observed_orcids":["https://orcid.org/0000-0001-8536-7175","https://orcid.org/0009-0008-9178-7369"]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuchen Zhang","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026363324","display_name":"Jude Fernandes","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jude Fernandes","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022730878","display_name":"Jane Dwivedi-Yu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jane Dwivedi-Yu","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Eleonora Presani","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eleonora Presani","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062696000","display_name":"Adina Williams","orcid":"https://orcid.org/0000-0001-5281-3343","observed_orcids":["https://orcid.org/0000-0001-5281-3343"]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Adina Williams","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"last","author":{"id":null,"display_name":"Eric Smith","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Eric Smith","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.4403,"has_fulltext":true,"cited_by_count":49,"citation_normalized_percentile":{"value":0.9724838,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"3764","last_page":"3814"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.7153000235557556,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.7153000235557556,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1177000030875206,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.07029999792575836,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bias-assessment","display_name":"bias assessment","score":0.7850000262260437},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.949999988079071},{"id":"https://openalex.org/keywords/generative-language-models","display_name":"generative language models","score":0.8299999833106995},{"id":"https://openalex.org/keywords/robustness","display_name":"robustness","score":0.5960000157356262},{"id":"https://openalex.org/keywords/algorithmic-fairness","display_name":"algorithmic fairness","score":0.6990000009536743}],"concepts":[{"id":"https://openalex.org/C2776002260","wikidata":"https://www.wikidata.org/wiki/Q245622","display_name":"Adina","level":3,"score":0.6294790506362915},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.5874277949333191},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.48933690786361694},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44740936160087585},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.41458773612976074},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.4141943156719208},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.380722314119339},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.3435057997703552},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2559322118759155},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.2321774959564209},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2232799530029297},{"id":"https://openalex.org/C66938386","wikidata":"https://www.wikidata.org/wiki/Q633538","display_name":"Structural engineering","level":1,"score":0.0},{"id":"https://openalex.org/C135628077","wikidata":"https://www.wikidata.org/wiki/Q220184","display_name":"Finite element method","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2023.emnlp-main.230","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.230","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-main.230","pdf_url":"https://aclanthology.org/2023.emnlp-main.230.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2311.18140","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.18140","pdf_url":"https://arxiv.org/pdf/2311.18140","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2311.18140","doi":"https://doi.org/10.48550/arxiv.2311.18140","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2311.18140","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.emnlp-main.230","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.230","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-main.230","pdf_url":"https://aclanthology.org/2023.emnlp-main.230.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389524209.pdf","grobid_xml":"https://content.openalex.org/works/W4389524209.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W2038466428","https://openalex.org/W2525332836","https://openalex.org/W2922862135","https://openalex.org/W2965373594","https://openalex.org/W3019416653","https://openalex.org/W3035591180","https://openalex.org/W3089430725","https://openalex.org/W3104748221","https://openalex.org/W3158733382","https://openalex.org/W3172205429","https://openalex.org/W3184144760","https://openalex.org/W3185341429","https://openalex.org/W3197577761","https://openalex.org/W3206487987","https://openalex.org/W4206500717","https://openalex.org/W4226278401","https://openalex.org/W4229005866","https://openalex.org/W4287179329","https://openalex.org/W4287890642","https://openalex.org/W4290943938","https://openalex.org/W4304731097","https://openalex.org/W4311991106","https://openalex.org/W4316135709","https://openalex.org/W4318719246","https://openalex.org/W4320003957","https://openalex.org/W4320830123","https://openalex.org/W4321161959","https://openalex.org/W4362655426","https://openalex.org/W4378189609","https://openalex.org/W4380319657","https://openalex.org/W4382239810","https://openalex.org/W4384918448","https://openalex.org/W4385572819","https://openalex.org/W4385573550","https://openalex.org/W4386566857"],"related_works":["https://openalex.org/W6885015010","https://openalex.org/W4385570061","https://openalex.org/W4402671800","https://openalex.org/W4389524534","https://openalex.org/W4404784221","https://openalex.org/W4285199616","https://openalex.org/W3168901128","https://openalex.org/W4412886926","https://openalex.org/W3118388622","https://openalex.org/W7131779856"],"abstract_inverted_index":{"David":[0],"Esiobu,":[1],"Xiaoqing":[2],"Tan,":[3],"Saghar":[4],"Hosseini,":[5],"Megan":[6],"Ung,":[7],"Yuchen":[8],"Zhang,":[9],"Jude":[10],"Fernandes,":[11],"Jane":[12],"Dwivedi-Yu,":[13],"Eleonora":[14],"Presani,":[15],"Adina":[16],"Williams,":[17],"Eric":[18],"Smith.":[19],"Proceedings":[20],"of":[21],"the":[22],"2023":[23],"Conference":[24],"on":[25],"Empirical":[26],"Methods":[27],"in":[28],"Natural":[29],"Language":[30],"Processing.":[31],"2023.":[32]},"counts_by_year":[{"year":2026,"cited_by_count":20},{"year":2025,"cited_by_count":19},{"year":2024,"cited_by_count":10}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2023-12-11T00:00:00"}