{"id":"https://openalex.org/W4405711033","doi":"https://doi.org/10.1162/tacl_a_00719","title":"Robust Pronoun Fidelity with English LLMs: Are they Reasoning, Repeating, or Just Biased?","display_name":"Robust Pronoun Fidelity with English LLMs: Are they Reasoning, Repeating, or Just Biased?","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4405711033","doi":"https://doi.org/10.1162/tacl_a_00719"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00719","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00719","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1162/tacl_a_00719","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013587924","display_name":"Vagrant Gautam","orcid":"https://orcid.org/0000-0002-7263-8578"},"institutions":[{"id":"https://openalex.org/I91712215","display_name":"Saarland University","ror":"https://ror.org/01jdpyv68","country_code":"DE","type":"education","lineage":["https://openalex.org/I91712215"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Vagrant Gautam","raw_affiliation_strings":["Saarland University, Germany. vgautam@lsv.uni-saarland.de"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Saarland University, Germany. vgautam@lsv.uni-saarland.de","institution_ids":["https://openalex.org/I91712215"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5095098417","display_name":"Eileen Bingert","orcid":null},"institutions":[{"id":"https://openalex.org/I91712215","display_name":"Saarland University","ror":"https://ror.org/01jdpyv68","country_code":"DE","type":"education","lineage":["https://openalex.org/I91712215"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Eileen Bingert","raw_affiliation_strings":["Saarland University, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Saarland University, Germany","institution_ids":["https://openalex.org/I91712215"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103158330","display_name":"Dawei Zhu","orcid":"https://orcid.org/0000-0003-1534-8854"},"institutions":[{"id":"https://openalex.org/I91712215","display_name":"Saarland University","ror":"https://ror.org/01jdpyv68","country_code":"DE","type":"education","lineage":["https://openalex.org/I91712215"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Dawei Zhu","raw_affiliation_strings":["Saarland University, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Saarland University, Germany","institution_ids":["https://openalex.org/I91712215"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022688200","display_name":"Anne Lauscher","orcid":"https://orcid.org/0000-0001-8590-9827"},"institutions":[{"id":"https://openalex.org/I159176309","display_name":"Universit\u00e4t Hamburg","ror":"https://ror.org/00g30e956","country_code":"DE","type":"education","lineage":["https://openalex.org/I159176309"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Anne Lauscher","raw_affiliation_strings":["Data Science Group, University of Hamburg, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Data Science Group, University of Hamburg, Germany","institution_ids":["https://openalex.org/I159176309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008875255","display_name":"Dietrich Klakow","orcid":"https://orcid.org/0000-0002-4147-9690"},"institutions":[{"id":"https://openalex.org/I91712215","display_name":"Saarland University","ror":"https://ror.org/01jdpyv68","country_code":"DE","type":"education","lineage":["https://openalex.org/I91712215"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Dietrich Klakow","raw_affiliation_strings":["Saarland University, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Saarland University, Germany","institution_ids":["https://openalex.org/I91712215"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.5557,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.7376934,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":97},"biblio":{"volume":"12","issue":null,"first_page":"1755","last_page":"1779"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9879999756813049,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10759","display_name":"Translation Studies and Practices","score":0.98580002784729,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8000550866127014},{"id":"https://openalex.org/keywords/fidelity","display_name":"Fidelity","score":0.6268450021743774},{"id":"https://openalex.org/keywords/pronoun","display_name":"Pronoun","score":0.598639190196991},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5065747499465942},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.49404552578926086},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.35567522048950195},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.07715708017349243}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8000550866127014},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.6268450021743774},{"id":"https://openalex.org/C2778551981","wikidata":"https://www.wikidata.org/wiki/Q36224","display_name":"Pronoun","level":2,"score":0.598639190196991},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5065747499465942},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49404552578926086},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.35567522048950195},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.07715708017349243},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1162/tacl_a_00719","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00719","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00719","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00719","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":84,"referenced_works":["https://openalex.org/W2124741472","https://openalex.org/W2239778283","https://openalex.org/W2466175319","https://openalex.org/W2555344324","https://openalex.org/W2559655401","https://openalex.org/W2799051177","https://openalex.org/W2893425640","https://openalex.org/W2914855263","https://openalex.org/W2920114910","https://openalex.org/W2926555354","https://openalex.org/W2963078909","https://openalex.org/W2963341956","https://openalex.org/W2963457723","https://openalex.org/W2963526187","https://openalex.org/W2963925965","https://openalex.org/W2964120396","https://openalex.org/W2965373594","https://openalex.org/W2970946372","https://openalex.org/W2972413484","https://openalex.org/W2979826702","https://openalex.org/W2980481841","https://openalex.org/W2982521860","https://openalex.org/W3024308166","https://openalex.org/W3034515982","https://openalex.org/W3034723486","https://openalex.org/W3034775979","https://openalex.org/W3035172163","https://openalex.org/W3037831233","https://openalex.org/W3133702157","https://openalex.org/W3154654049","https://openalex.org/W3177189402","https://openalex.org/W3178522238","https://openalex.org/W3197577761","https://openalex.org/W3198409578","https://openalex.org/W3211511509","https://openalex.org/W3212978464","https://openalex.org/W4212774754","https://openalex.org/W4221154823","https://openalex.org/W4229005866","https://openalex.org/W4285287265","https://openalex.org/W4287887365","https://openalex.org/W4287887849","https://openalex.org/W4287889150","https://openalex.org/W4287891150","https://openalex.org/W4304730661","https://openalex.org/W4307079201","https://openalex.org/W4365799947","https://openalex.org/W4377157114","https://openalex.org/W4384918448","https://openalex.org/W4385565605","https://openalex.org/W4385569782","https://openalex.org/W4385570581","https://openalex.org/W4385570682","https://openalex.org/W4385572155","https://openalex.org/W4385574268","https://openalex.org/W4385893893","https://openalex.org/W4389518729","https://openalex.org/W4389518756","https://openalex.org/W4389518805","https://openalex.org/W4391876619","https://openalex.org/W4393177975","https://openalex.org/W4393968405","https://openalex.org/W4401043206","https://openalex.org/W4402683949","https://openalex.org/W6635469476","https://openalex.org/W6675366379","https://openalex.org/W6721933647","https://openalex.org/W6760413209","https://openalex.org/W6766673545","https://openalex.org/W6768021236","https://openalex.org/W6809616987","https://openalex.org/W6809845343","https://openalex.org/W6811340617","https://openalex.org/W6838865847","https://openalex.org/W6845581020","https://openalex.org/W6846556436","https://openalex.org/W6847076894","https://openalex.org/W6849594959","https://openalex.org/W6851579256","https://openalex.org/W6851762504","https://openalex.org/W6854866820","https://openalex.org/W6857660365","https://openalex.org/W6859914819","https://openalex.org/W6862954833"],"related_works":["https://openalex.org/W2381850946","https://openalex.org/W4380449851","https://openalex.org/W4205961602","https://openalex.org/W3125091513","https://openalex.org/W4318832338","https://openalex.org/W2035216435","https://openalex.org/W2072686876","https://openalex.org/W4248383205","https://openalex.org/W1594011529","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Abstract":[0],"Robust,":[1],"faithful,":[2],"and":[3,58,88,102,104,136,195],"harm-free":[4],"pronoun":[5,49,67,84,122,155,170],"use":[6,19],"for":[7,13],"individuals":[8],"is":[9,62,111,172],"an":[10,109],"important":[11],"goal":[12],"language":[14],"model":[15,92,209],"development":[16],"as":[17],"their":[18],"increases,":[20],"but":[21,127],"prior":[22],"work":[23],"tends":[24],"to":[25,63,81,158,189,196],"study":[26],"only":[27],"one":[28,150],"or":[29],"two":[30],"of":[31,48,76,208],"these":[32],"characteristics":[33],"at":[34],"a":[35,52,55,72,114,153,176],"time.":[36],"To":[37],"measure":[38,82],"progress":[39],"towards":[40],"the":[41,46,60,65,124,191],"combined":[42],"goal,":[43],"we":[44,89,193],"introduce":[45],"task":[47,61],"fidelity:":[50],"Given":[51],"context":[53],"introducing":[54],"co-referring":[56],"entity":[57],"pronoun,":[59,115],"reuse":[64,120],"correct":[66],"later.":[68],"We":[69,186],"present":[70],"RUFF,":[71],"carefully":[73,197],"designed":[74],"dataset":[75],"over":[77],"5":[78],"million":[79],"instances":[80],"robust":[83],"fidelity":[85,171],"in":[86,123,175,200],"English,":[87],"evaluate":[90,198],"37":[91],"variants":[93],"from":[94],"nine":[95],"popular":[96],"families,":[97],"across":[98],"architectures":[99],"(encoder-only,":[100],"decoder-only,":[101],"encoder-decoder)":[103],"scales":[105],"(11M-70B":[106],"parameters).":[107],"When":[108],"individual":[110],"introduced":[112],"with":[113,132,152],"models":[116,139],"can":[117],"mostly":[118],"faithfully":[119],"this":[121],"next":[125],"sentence,":[126],"they":[128],"are":[129,140],"significantly":[130],"worse":[131],"she/her/her,":[133],"singular":[134],"they,":[135],"neopronouns.":[137],"Moreover,":[138],"easily":[141],"distracted":[142],"by":[143,162],"non-adversarial":[144],"sentences":[145],"discussing":[146],"other":[147],"people;":[148],"even":[149],"sentence":[151],"distractor":[154],"causes":[156],"accuracy":[157],"drop":[159],"on":[160],"average":[161],"34":[163],"percentage":[164],"points.":[165],"Our":[166],"results":[167],"show":[168],"that":[169],"not":[173],"robust,":[174],"simple,":[177],"naturalistic":[178],"setting":[179],"where":[180,202],"humans":[181],"achieve":[182],"nearly":[183],"100%":[184],"accuracy.":[185],"encourage":[187],"researchers":[188],"bridge":[190],"gaps":[192],"find":[194],"reasoning":[199],"settings":[201],"superficial":[203],"repetition":[204],"might":[205],"inflate":[206],"perceptions":[207],"performance.":[210]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}