{"id":"https://openalex.org/W7139108591","doi":"https://doi.org/10.1609/aaai.v40i44.41155","title":"Can LLMs Detect Their Confabulations? Estimating Reliability in Uncertainty-Aware Language Models","display_name":"Can LLMs Detect Their Confabulations? Estimating Reliability in Uncertainty-Aware Language Models","publication_year":2026,"publication_date":"2026-03-14","ids":{"openalex":"https://openalex.org/W7139108591","doi":"https://doi.org/10.1609/aaai.v40i44.41155"},"language":null,"primary_location":{"id":"doi:10.1609/aaai.v40i44.41155","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i44.41155","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/41155/45116","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/41155/45116","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5096941638","display_name":"Tianyi Zhou","orcid":"https://orcid.org/0000-0001-9566-8035"},"institutions":[{"id":"https://openalex.org/I86987016","display_name":"KTH Royal Institute of Technology","ror":"https://ror.org/026vcq606","country_code":"SE","type":"education","lineage":["https://openalex.org/I86987016"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Tianyi Zhou","raw_affiliation_strings":["KTH Royal Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KTH Royal Institute of Technology","institution_ids":["https://openalex.org/I86987016"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046707698","display_name":"Johanne Medina","orcid":"https://orcid.org/0000-0003-0642-7681"},"institutions":[{"id":"https://openalex.org/I4210144839","display_name":"Hamad bin Khalifa University","ror":"https://ror.org/03eyq4y97","country_code":"QA","type":"education","lineage":["https://openalex.org/I4210144839"]}],"countries":["QA"],"is_corresponding":false,"raw_author_name":"Johanne Medina","raw_affiliation_strings":["Qatar Computing Research Institute, Hamad Bin Khalifa University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Qatar Computing Research Institute, Hamad Bin Khalifa University","institution_ids":["https://openalex.org/I4210144839"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037947876","display_name":"Sanjay Chawla","orcid":"https://orcid.org/0000-0002-8102-2572"},"institutions":[{"id":"https://openalex.org/I4210144839","display_name":"Hamad bin Khalifa University","ror":"https://ror.org/03eyq4y97","country_code":"QA","type":"education","lineage":["https://openalex.org/I4210144839"]}],"countries":["QA"],"is_corresponding":false,"raw_author_name":"Sanjay Chawla","raw_affiliation_strings":["Qatar Computing Research Institute, Hamad Bin Khalifa University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Qatar Computing Research Institute, Hamad Bin Khalifa University","institution_ids":["https://openalex.org/I4210144839"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.828585,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"40","issue":"44","first_page":"38164","last_page":"38172"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5784000158309937,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5784000158309937,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.14229999482631683,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.030300000682473183,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.7642999887466431},{"id":"https://openalex.org/keywords/salient","display_name":"Salient","score":0.6665999889373779},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6225000023841858},{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.5817999839782715},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.3630000054836273},{"id":"https://openalex.org/keywords/estimation","display_name":"Estimation","score":0.3508000075817108},{"id":"https://openalex.org/keywords/hidden-markov-model","display_name":"Hidden Markov model","score":0.30790001153945923}],"concepts":[{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.7642999887466431},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6779999732971191},{"id":"https://openalex.org/C2780719617","wikidata":"https://www.wikidata.org/wiki/Q1030752","display_name":"Salient","level":2,"score":0.6665999889373779},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6225000023841858},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.5817999839782715},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4318000078201294},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3939000070095062},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.38029998540878296},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3630000054836273},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.3508000075817108},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.30790001153945923},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.3075000047683716},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.305400013923645},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.28360000252723694},{"id":"https://openalex.org/C32230216","wikidata":"https://www.wikidata.org/wiki/Q7882499","display_name":"Uncertainty quantification","level":2,"score":0.28189998865127563},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.27129998803138733},{"id":"https://openalex.org/C61224824","wikidata":"https://www.wikidata.org/wiki/Q2260434","display_name":"Mixture model","level":2,"score":0.2711000144481659},{"id":"https://openalex.org/C201729545","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability theory","level":3,"score":0.26339998841285706},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.26019999384880066},{"id":"https://openalex.org/C12174686","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk assessment","level":2,"score":0.25760000944137573}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1609/aaai.v40i44.41155","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i44.41155","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/41155/45116","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v40i44.41155","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i44.41155","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/41155/45116","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.6799085140228271}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7139108591.pdf","grobid_xml":"https://content.openalex.org/works/W7139108591.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"are":[4],"prone":[5],"to":[6,59,77],"generating":[7],"fluent":[8],"but":[9],"incorrect":[10,119],"content,":[11],"known":[12],"as":[13,29],"confabulation,":[14],"which":[15],"poses":[16],"increasing":[17],"risks":[18],"in":[19,134],"multi-turn":[20],"or":[21],"agentic":[22],"applications":[23],"where":[24],"outputs":[25,143],"may":[26],"be":[27],"reused":[28],"context.":[30],"In":[31],"this":[32],"work,":[33],"we":[34,68,100],"investigate":[35],"how":[36],"in-context":[37,104],"information":[38,105],"influences":[39],"model":[40,65,111,135],"behavior":[41,136],"and":[42,71,81,110,126,137,157],"whether":[43],"LLMs":[44],"can":[45],"identify":[46,78],"their":[47,83],"unreliable":[48,142],"responses.":[49],"We":[50],"propose":[51],"a":[52,122],"reliability":[53,91],"estimation":[54],"that":[55,102],"leverages":[56],"token-level":[57],"uncertainty":[58,73,125,155],"guide":[60],"the":[61,139,151,159],"aggregation":[62],"of":[63,141,153,161],"internal":[64],"representations.":[66],"Specifically,":[67],"compute":[69],"aleatoric":[70],"epistemic":[72],"from":[74],"output":[75],"logits":[76],"salient":[79],"tokens":[80],"aggregate":[82],"hidden":[84],"states":[85],"into":[86],"compact":[87],"representations":[88],"for":[89,164],"response-level":[90],"prediction.":[92],"Through":[93],"controlled":[94],"experiments":[95],"on":[96],"open":[97],"QA":[98],"benchmarks,":[99],"find":[101],"correct":[103],"improves":[106,138],"both":[107],"answer":[108],"accuracy":[109],"confidence,":[112],"while":[113],"misleading":[114],"context":[115],"often":[116],"induces":[117],"confidently":[118],"responses,":[120],"revealing":[121],"misalignment":[123],"between":[124],"correctness.":[127],"Our":[128],"probing-based":[129],"method":[130],"captures":[131],"these":[132],"shifts":[133],"detection":[140],"across":[144],"multiple":[145],"open-source":[146],"LLMs.":[147],"These":[148],"results":[149],"underscore":[150],"limitations":[152],"direct":[154],"signals":[156],"highlight":[158],"potential":[160],"uncertainty-guided":[162],"probing":[163],"reliability-aware":[165],"generation.":[166]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-03-20T00:00:00"}