{"id":"https://openalex.org/W4417258166","doi":"https://doi.org/10.52202/085713-5729","title":"AbstentionBench: Reasoning LLMs Fail on Unanswerable Questions","display_name":"AbstentionBench: Reasoning LLMs Fail on Unanswerable Questions","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4417258166","doi":"https://doi.org/10.52202/085713-5729"},"language":"en","primary_location":{"id":"doi:10.52202/085713-5729","doi":"https://doi.org/10.52202/085713-5729","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-5729","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2506.09038","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5084173427","display_name":"Polina Kirichenko","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Polina Kirichenko","raw_affiliation_strings":["New York University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030917007","display_name":"Mark Ibrahim","orcid":"https://orcid.org/0009-0005-2135-7476","observed_orcids":["https://orcid.org/0009-0005-2135-7476"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mark Ibrahim","raw_affiliation_strings":["Fundamental AI Research (FAIR) at Meta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fundamental AI Research (FAIR) at Meta","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010790447","display_name":"Kamalika Chaudhuri","orcid":"https://orcid.org/0000-0001-9646-7710","observed_orcids":["https://orcid.org/0000-0001-9646-7710"]},"institutions":[{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kamalika Chaudhuri","raw_affiliation_strings":["FAIR, Meta and UCSD"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"FAIR, Meta and UCSD","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024504757","display_name":"Samuel J. Bell","orcid":"https://orcid.org/0000-0002-9437-5449","observed_orcids":["https://orcid.org/0000-0002-9437-5449"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Samuel J. Bell","raw_affiliation_strings":["Meta FAIR"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta FAIR","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.7482,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.95922616,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"190800","last_page":"190850"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8852999806404114,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8852999806404114,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.05849999934434891,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.02280000038444996,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/unanswerable-questions","display_name":"unanswerable questions","score":0.777999997138977},{"id":"https://openalex.org/keywords/reasoning-under-uncertainty","display_name":"uncertainty reasoning","score":0.8009999990463257},{"id":"https://openalex.org/keywords/benchmark","display_name":"benchmark","score":0.5040000081062317},{"id":"https://openalex.org/keywords/reasoning-llms","display_name":"reasoning LLMs","score":0.8059999942779541},{"id":"https://openalex.org/keywords/false-premises","display_name":"false premises","score":0.9070000052452087},{"id":"https://openalex.org/keywords/outdated-information","display_name":"outdated information","score":0.8029999732971191},{"id":"https://openalex.org/keywords/system-prompts","display_name":"system prompts","score":0.8069999814033508},{"id":"https://openalex.org/keywords/llm-reliability","display_name":"LLM reliability","score":0.8339999914169312}],"concepts":[{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.4693000018596649},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.45489999651908875},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.41339999437332153},{"id":"https://openalex.org/C118084267","wikidata":"https://www.wikidata.org/wiki/Q26110","display_name":"Positive economics","level":1,"score":0.40049999952316284},{"id":"https://openalex.org/C168725872","wikidata":"https://www.wikidata.org/wiki/Q991663","display_name":"Sophistication","level":2,"score":0.3269999921321869},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.3082999885082245},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2976999878883362},{"id":"https://openalex.org/C2776325391","wikidata":"https://www.wikidata.org/wiki/Q6917865","display_name":"Motivated reasoning","level":3,"score":0.28940001130104065},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.27160000801086426},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.2590000033378601},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2587999999523163},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.25200000405311584}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.52202/085713-5729","doi":"https://doi.org/10.52202/085713-5729","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-5729","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2506.09038","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.09038","pdf_url":"https://arxiv.org/pdf/2506.09038","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2506.09038","doi":"https://doi.org/10.48550/arxiv.2506.09038","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2506.09038","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2506.09038","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.09038","pdf_url":"https://arxiv.org/pdf/2506.09038","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7148867011","https://openalex.org/W6910763932","https://openalex.org/W6966698728","https://openalex.org/W7221539578","https://openalex.org/W7077901723","https://openalex.org/W7125350789","https://openalex.org/W4411934268","https://openalex.org/W7166133053","https://openalex.org/W4394947099","https://openalex.org/W4404782525"],"abstract_inverted_index":{"For":[0],"Large":[1],"Language":[2],"Models":[3],"(LLMs)":[4],"to":[5,18,39,49,171,178],"be":[6,31],"reliably":[7],"deployed":[8],"in":[9,121,162],"both":[10],"everyday":[11],"and":[12,43,91,104,140],"high-stakes":[13],"domains,":[14],"knowing":[15],"when":[16],"not":[17,166],"answer":[19,50],"is":[20,100,109],"equally":[21],"critical":[22],"as":[23],"answering":[24],"correctly.":[25],"Real-world":[26],"user":[27],"queries,":[28],"which":[29,144],"can":[30,159],"underspecified,":[32],"ill-posed,":[33],"or":[34],"fundamentally":[35],"unanswerable,":[36],"require":[37],"LLMs":[38,97,116],"reason":[40,172],"about":[41,173],"uncertainty":[42],"selectively":[44],"abstain":[45],"--":[46],"i.e.,":[47],"refuse":[48],"definitively.":[51],"However,":[52],"abstention":[53,76,99,132,161],"remains":[54],"understudied,":[55],"without":[56],"a":[57,70,154],"systematic":[58],"evaluation":[59],"framework":[60],"for":[61,73,138],"modern":[62],"LLMs.":[63],"In":[64],"this":[65],"work,":[66],"we":[67,126],"introduce":[68],"AbstentionBench,":[69],"large-scale":[71],"benchmark":[72],"holistically":[74],"evaluating":[75],"across":[77],"20":[78,95],"diverse":[79],"datasets,":[80],"including":[81],"questions":[82],"with":[83],"unknown":[84],"answers,":[85],"underspecification,":[86],"false":[87],"premises,":[88],"subjective":[89],"interpretations,":[90],"outdated":[92],"information.":[93],"Evaluating":[94],"frontier":[96],"reveals":[98],"an":[101],"unsolved":[102],"problem,":[103],"one":[105],"where":[106],"scaling":[107],"models":[108,146],"of":[110],"little":[111],"use.":[112],"While":[113],"recent":[114],"reasoning":[115,129,145],"have":[117],"shown":[118],"impressive":[119],"results":[120],"complex":[122],"problem":[123],"solving,":[124],"surprisingly,":[125],"find":[127,151],"that":[128,152],"fine-tuning":[130],"degrades":[131],"(by":[133],"$24\\%$":[134],"on":[135,143],"average),":[136],"even":[137],"math":[139],"science":[141],"domains":[142],"are":[147],"explicitly":[148],"trained.":[149],"We":[150,175],"while":[153],"carefully":[155],"crafted":[156],"system":[157],"prompt":[158],"boost":[160],"practice,":[163],"it":[164],"does":[165],"resolve":[167],"models'":[168],"fundamental":[169],"inability":[170],"uncertainty.":[174],"release":[176],"AbstentionBench":[177],"foster":[179],"research":[180],"into":[181],"advancing":[182],"LLM":[183],"reliability.":[184]},"counts_by_year":[{"year":2026,"cited_by_count":5}],"updated_date":"2026-10-29T09:56:32.521200","created_date":"2025-10-10T00:00:00"}