{"id":"https://openalex.org/W7160985265","doi":"https://doi.org/10.48550/arxiv.2605.11330","title":"Rethinking Evaluation for LLM Hallucination Detection: A Desiderata, A New RAG-based Benchmark, New Insights","display_name":"Rethinking Evaluation for LLM Hallucination Detection: A Desiderata, A New RAG-based Benchmark, New Insights","publication_year":2026,"publication_date":"2026-05-11","ids":{"openalex":"https://openalex.org/W7160985265","doi":"https://doi.org/10.48550/arxiv.2605.11330"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.11330","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.11330","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.11330","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136058316","display_name":"Wenbo Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Wenbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136008004","display_name":"Veena Padmanabhan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Padmanabhan, Veena","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074127440","display_name":"Tootiya Giyahchi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Giyahchi, Tootiya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136064478","display_name":"Elaine Wong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wong, Elaine","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136085992","display_name":"Leman Akoglu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Akoglu, Leman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.2540999948978424,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.2540999948978424,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.08030000329017639,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.04580000042915344,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8104000091552734},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.6976000070571899},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6568999886512756},{"id":"https://openalex.org/keywords/detector","display_name":"Detector","score":0.47929999232292175},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.4690000116825104},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.4212999939918518},{"id":"https://openalex.org/keywords/background-noise","display_name":"Background noise","score":0.32899999618530273}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8104000091552734},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.6976000070571899},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6568999886512756},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6536999940872192},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4851999878883362},{"id":"https://openalex.org/C94915269","wikidata":"https://www.wikidata.org/wiki/Q1834857","display_name":"Detector","level":2,"score":0.47929999232292175},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.4690000116825104},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4657000005245209},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.4212999939918518},{"id":"https://openalex.org/C100675267","wikidata":"https://www.wikidata.org/wiki/Q1371624","display_name":"Background noise","level":2,"score":0.32899999618530273},{"id":"https://openalex.org/C29265498","wikidata":"https://www.wikidata.org/wiki/Q7047719","display_name":"Noise measurement","level":3,"score":0.32440000772476196},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.3192000091075897},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.3140999972820282},{"id":"https://openalex.org/C2777489069","wikidata":"https://www.wikidata.org/wiki/Q1589822","display_name":"Ceiling (cloud)","level":2,"score":0.29829999804496765},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2906999886035919},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2897999882698059},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.28110000491142273},{"id":"https://openalex.org/C101068787","wikidata":"https://www.wikidata.org/wiki/Q475042","display_name":"Experimental psychology","level":3,"score":0.2721000015735626},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.257999986410141},{"id":"https://openalex.org/C163294075","wikidata":"https://www.wikidata.org/wiki/Q581861","display_name":"Noise reduction","level":2,"score":0.2533000111579895}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.11330","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.11330","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.11330","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.11330","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Hallucination,":[0],"broadly":[1],"referring":[2],"to":[3,24,53,123,215],"unfaithful,":[4],"fabricated,":[5],"or":[6,125],"inconsistent":[7],"content":[8],"generated":[9],"by":[10],"LLMs,":[11],"has":[12,21],"wide-ranging":[13],"implications.":[14],"Therefore,":[15],"a":[16,44,136,145],"large":[17],"body":[18],"of":[19,46,67,73,178],"effort":[20],"been":[22],"devoted":[23],"detecting":[25],"LLM":[26],"hallucinations,":[27],"as":[28,30],"well":[29],"designing":[31],"benchmark":[32,152,246],"datasets":[33],"for":[34,48,55,111,212,258],"evaluating":[35],"these":[36,130],"detectors.":[37],"In":[38],"this":[39],"work,":[40],"we":[41,132,172,190],"first":[42],"establish":[43],"desiderata":[45,69],"properties":[47,156],"hallucination":[49,256],"detection":[50,235,257],"benchmarks":[51,87,103],"(HDBs)":[52],"exhibit":[54],"effective":[56],"evaluation.":[57],"A":[58],"critical":[59],"look":[60],"at":[61],"existing":[62],"HDBs":[63],"through":[64],"the":[65,77,164,168,217,224],"lens":[66],"our":[68,151,197,240,244],"reveals":[70],"that":[71,143,204,239],"none":[72],"them":[74],"exhibits":[75,153],"all":[76,154],"properties.":[78],"We":[79,237],"identify":[80],"two":[81],"largest":[82],"gaps:":[83],"(1)":[84,158],"RAG-based":[85,138,194,221,259],"grounded":[86],"with":[88,119,163,181,243],"long":[89],"context":[90,166],"are":[91],"severely":[92],"lacking":[93],"(partly":[94],"because":[95],"length":[96],"impedes":[97],"human":[98,124,147],"annotation);":[99],"and":[100,134,170,174,185,230,251],"(2)":[101,171],"Existing":[102],"do":[104],"not":[105],"make":[106],"available":[107],"realistic":[108],"label":[109,120,232],"noise":[110,121,187,233],"stress-testing":[112],"detectors":[113,203,214],"although":[114],"real-world":[115],"use-cases":[116],"often":[117],"grapple":[118],"due":[122],"automated/weak":[126],"annotation.":[127],"To":[128],"close":[129],"gaps,":[131],"build":[133],"open-source":[135],"new":[137,206],"HDB":[139],"called":[140],"T":[141,159,198],"RIVIA+":[142,160],"underwent":[144],"rigorous":[146],"annotation":[148],"process.":[149],"Notably,":[150],"desirable":[155],"including":[157,196],"contains":[161],"samples":[162],"longest":[165],"in":[167],"literature;":[169],"design":[173],"share":[175],"four":[176],"sets":[177],"noisy":[179],"labels":[180],"different,":[182],"both":[183],"sample-dependent":[184],"sampleindependent,":[186],"schemes.":[188],"Finally,":[189],"perform":[191],"experiments":[192],"on":[193,220,255],"HDBs,":[195,222],"RIVIA+,":[199],"using":[200],"popular":[201],"SOTA":[202],"reveal":[205],"insights:":[207],"(i)":[208],"ample":[209],"room":[210],"remains":[211],"current":[213],"reach":[216],"performance":[218],"ceiling":[219],"(ii)":[223],"basic":[225],"LLM-as-a-Judge":[226],"baseline":[227],"performs":[228],"competitively,":[229],"(iii)":[231],"hinders":[234],"performance.":[236],"expect":[238],"findings,":[241],"along":[242],"proposed":[245],"1":[247],",":[248],"will":[249],"motivate":[250],"foster":[252],"needed":[253],"research":[254],"tasks.":[260]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-14T00:00:00"}