{"id":"https://openalex.org/W7166822101","doi":"https://doi.org/10.18653/v1/2026.acl-long.680","title":"Rethinking Evaluation for LLM Hallucination Detection: A Desiderata, A New RAG-based Benchmark, New Insights","display_name":"Rethinking Evaluation for LLM Hallucination Detection: A Desiderata, A New RAG-based Benchmark, New Insights","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166822101","doi":"https://doi.org/10.18653/v1/2026.acl-long.680"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2026.acl-long.680","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.680","pdf_url":"https://aclanthology.org/2026.acl-long.680.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.acl-long.680.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139822403","display_name":"Wenbo Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wenbo Chen","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139799830","display_name":"Veena Padmanabhan","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Veena Padmanabhan","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074127440","display_name":"Tootiya Giyahchi","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tootiya Giyahchi","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044922102","display_name":"Elaine Wong","orcid":"https://orcid.org/0000-0002-2561-3482"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Elaine Wong","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5139848141","display_name":"Leman Akoglu","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Leman Akoglu","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.87788533,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"14912","last_page":"14931"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12553","display_name":"Psychedelics and Drug Studies","score":0.41600000858306885,"subfield":{"id":"https://openalex.org/subfields/3203","display_name":"Clinical Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12553","display_name":"Psychedelics and Drug Studies","score":0.41600000858306885,"subfield":{"id":"https://openalex.org/subfields/3203","display_name":"Clinical Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13397","display_name":"Hallucinations in medical conditions","score":0.1324000060558319,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12536","display_name":"Topological and Geometric Data Analysis","score":0.023499999195337296,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.33869999647140503},{"id":"https://openalex.org/keywords/delusion","display_name":"Delusion","score":0.289900004863739},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.2603999972343445},{"id":"https://openalex.org/keywords/identity","display_name":"Identity (music)","score":0.2540999948978424},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.2515999972820282}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5375999808311462},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.33869999647140503},{"id":"https://openalex.org/C11171543","wikidata":"https://www.wikidata.org/wiki/Q41630","display_name":"Psychoanalysis","level":1,"score":0.3149000108242035},{"id":"https://openalex.org/C2779043874","wikidata":"https://www.wikidata.org/wiki/Q189643","display_name":"Delusion","level":2,"score":0.289900004863739},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.28859999775886536},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.2603999972343445},{"id":"https://openalex.org/C2778355321","wikidata":"https://www.wikidata.org/wiki/Q17079427","display_name":"Identity (music)","level":2,"score":0.2540999948978424},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.2515999972820282},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.24869999289512634},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.24539999663829803}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.acl-long.680","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.680","pdf_url":"https://aclanthology.org/2026.acl-long.680.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.acl-long.680","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.680","pdf_url":"https://aclanthology.org/2026.acl-long.680.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Gender equality","id":"https://metadata.un.org/sdg/5","score":0.5327463746070862}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166822101.pdf","grobid_xml":"https://content.openalex.org/works/W7166822101.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Hallucination,":[0],"broadly":[1],"referring":[2],"to":[3,23,51,119,204],"unfaithful,":[4],"fabricated,":[5],"or":[6,121],"inconsistent":[7],"content":[8],"generated":[9],"by":[10],"LLMs,":[11],"has":[12,20],"wide-ranging":[13],"implications.Therefore,":[14],"a":[15,42,131,139],"large":[16],"body":[17],"of":[18,44,64,70,169],"effort":[19],"been":[21],"devoted":[22],"detecting":[24],"LLM":[25],"hallucinations,":[26],"as":[27,29],"well":[28],"designing":[30],"benchmark":[31,145,234,253],"datasets":[32],"for":[33,46,53,107,201,246],"evaluating":[34],"these":[35,125],"detectors.In":[36],"this":[37],"work,":[38],"we":[39,127,163,180],"first":[40],"establish":[41],"desiderata":[43,66],"properties":[45,149],"hallucination":[47,244],"detection":[48,224,245],"benchmarks":[49,83,99],"(HDBs)":[50],"exhibit":[52],"effective":[54],"evaluation.A":[55],"critical":[56],"look":[57],"at":[58,258],"existing":[59],"HDBs":[60],"through":[61],"the":[62,74,156,160,206,213,252],"lens":[63],"our":[65,144,187,228,232],"reveals":[67],"that":[68,137,193,227],"none":[69],"them":[71],"exhibits":[72,146],"all":[73,147],"properties.We":[75],"identify":[76],"two":[77],"largest":[78],"gaps:":[79],"(1)":[80,151],"RAG-based":[81,133,184,210,247],"grounded":[82],"with":[84,115,155,172,231,255],"long":[85],"context":[86,158],"are":[87],"severely":[88],"lacking":[89],"(partly":[90],"because":[91],"length":[92],"impedes":[93],"human":[94,120,141],"annotation);":[95],"and":[96,129,165,176,219,239],"(2)":[97],"Existing":[98],"do":[100],"not":[101],"make":[102],"available":[103],"realistic":[104],"label":[105,116,221],"noise":[106,117,178,222],"stress-testing":[108],"detectors":[109,192,203],"although":[110],"real-world":[111],"use-cases":[112],"often":[113],"grapple":[114],"due":[118],"automated/weak":[122],"annotation.To":[123],"close":[124],"gaps,":[126],"build":[128],"open-source":[130],"new":[132,195],"HDB":[134],"called":[135],"TRIVIA+":[136,152],"underwent":[138],"rigorous":[140],"annotation":[142],"process.Notably,":[143],"desirable":[148],"including":[150,186],"contains":[153],"samples":[154],"longest":[157],"in":[159],"literature;":[161],"and(2)":[162],"design":[164],"share":[166],"four":[167],"sets":[168],"noisy":[170],"labels":[171],"different,":[173],"both":[174],"sample-dependent":[175],"sampleindependent,":[177],"schemes.Finally,":[179],"perform":[181],"experiments":[182],"on":[183,209,243],"HDBs,":[185,211],"TRIVIA+,":[188],"using":[189],"popular":[190],"SOTA":[191],"reveal":[194],"insights:":[196],"(i)":[197],"ample":[198],"room":[199],"remains":[200],"current":[202],"reach":[205],"performance":[207],"ceiling":[208],"(ii)":[212],"basic":[214],"LLM-as-a-Judge":[215],"baseline":[216],"performs":[217],"competitively,":[218],"(iii)":[220],"hinders":[223],"performance.We":[225],"expect":[226],"findings,":[229],"along":[230],"proposed":[233],"1":[235,249],",":[236],"will":[237],"motivate":[238],"foster":[240],"needed":[241],"research":[242],"tasks.":[248],"We":[250],"release":[251],"dataset":[254],"expert":[256],"annotations":[257],"github.com/Wenbo11/TriviaPlus.":[259]},"counts_by_year":[],"updated_date":"2026-08-24T07:32:12.397491","created_date":"2026-07-02T00:00:00"}