{"id":"https://openalex.org/W4415683709","doi":"https://doi.org/10.3390/info16110937","title":"DefAn: Definitive Answer Dataset for LLM Hallucination Evaluation","display_name":"DefAn: Definitive Answer Dataset for LLM Hallucination Evaluation","publication_year":2025,"publication_date":"2025-10-28","ids":{"openalex":"https://openalex.org/W4415683709","doi":"https://doi.org/10.3390/info16110937"},"language":"en","primary_location":{"id":"doi:10.3390/info16110937","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info16110937","pdf_url":"https://www.mdpi.com/2078-2489/16/11/937/pdf?version=1761653353","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2078-2489/16/11/937/pdf?version=1761653353","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005860166","display_name":"Asif Rahman","orcid":"https://orcid.org/0000-0003-1375-4330"},"institutions":[{"id":"https://openalex.org/I134085113","display_name":"King Fahd University of Petroleum and Minerals","ror":"https://ror.org/03yez3163","country_code":"SA","type":"education","lineage":["https://openalex.org/I134085113"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"A. B. M. Ashikur Rahman","raw_affiliation_strings":["Information and Computer Science Department, King Fahd University of Petroleum and Minerals, Dhahran 31261, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0003-1375-4330","affiliations":[{"raw_affiliation_string":"Information and Computer Science Department, King Fahd University of Petroleum and Minerals, Dhahran 31261, Saudi Arabia","institution_ids":["https://openalex.org/I134085113"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084633048","display_name":"Saeed Anwar","orcid":"https://orcid.org/0000-0002-0692-8411"},"institutions":[{"id":"https://openalex.org/I134085113","display_name":"King Fahd University of Petroleum and Minerals","ror":"https://ror.org/03yez3163","country_code":"SA","type":"education","lineage":["https://openalex.org/I134085113"]},{"id":"https://openalex.org/I177877127","display_name":"The University of Western Australia","ror":"https://ror.org/047272k79","country_code":"AU","type":"education","lineage":["https://openalex.org/I177877127"]}],"countries":["AU","SA"],"is_corresponding":true,"raw_author_name":"Saeed Anwar","raw_affiliation_strings":["Department of Computer Science and Software Engineering, University of Western Australia, 35 Stirling Highway, Perth 6009, Australia","Information and Computer Science Department, King Fahd University of Petroleum and Minerals, Dhahran 31261, Saudi Arabia","SDAIA-KFUPM Joint Research Center for Artificial Intelligence (JRC-AI), Dhahran 31261, Saudi Arabia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Software Engineering, University of Western Australia, 35 Stirling Highway, Perth 6009, Australia","institution_ids":["https://openalex.org/I177877127"]},{"raw_affiliation_string":"Information and Computer Science Department, King Fahd University of Petroleum and Minerals, Dhahran 31261, Saudi Arabia","institution_ids":["https://openalex.org/I134085113"]},{"raw_affiliation_string":"SDAIA-KFUPM Joint Research Center for Artificial Intelligence (JRC-AI), Dhahran 31261, Saudi Arabia","institution_ids":["https://openalex.org/I134085113"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003364006","display_name":"Muhammad Usman","orcid":"https://orcid.org/0000-0003-2059-7206"},"institutions":[{"id":"https://openalex.org/I39470171","display_name":"Ontario Tech University","ror":"https://ror.org/016zre027","country_code":"CA","type":"education","lineage":["https://openalex.org/I39470171"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Muhammad Usman","raw_affiliation_strings":["Faculty of Science, Ontario Tech University, 2000 Simcoe Street North Oshawa, Oshawa, ON L1G 0C5, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Faculty of Science, Ontario Tech University, 2000 Simcoe Street North Oshawa, Oshawa, ON L1G 0C5, Canada","institution_ids":["https://openalex.org/I39470171"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070341092","display_name":"Irfan Ahmad","orcid":"https://orcid.org/0000-0001-8311-1731"},"institutions":[{"id":"https://openalex.org/I134085113","display_name":"King Fahd University of Petroleum and Minerals","ror":"https://ror.org/03yez3163","country_code":"SA","type":"education","lineage":["https://openalex.org/I134085113"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Irfan Ahmad","raw_affiliation_strings":["Information and Computer Science Department, King Fahd University of Petroleum and Minerals, Dhahran 31261, Saudi Arabia","SDAIA-KFUPM Joint Research Center for Artificial Intelligence (JRC-AI), Dhahran 31261, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0001-8311-1731","affiliations":[{"raw_affiliation_string":"Information and Computer Science Department, King Fahd University of Petroleum and Minerals, Dhahran 31261, Saudi Arabia","institution_ids":["https://openalex.org/I134085113"]},{"raw_affiliation_string":"SDAIA-KFUPM Joint Research Center for Artificial Intelligence (JRC-AI), Dhahran 31261, Saudi Arabia","institution_ids":["https://openalex.org/I134085113"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5089986388","display_name":"Ajmal Mian","orcid":"https://orcid.org/0000-0002-5206-3842"},"institutions":[{"id":"https://openalex.org/I177877127","display_name":"The University of Western Australia","ror":"https://ror.org/047272k79","country_code":"AU","type":"education","lineage":["https://openalex.org/I177877127"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Ajmal Mian","raw_affiliation_strings":["Department of Computer Science and Software Engineering, University of Western Australia, 35 Stirling Highway, Perth 6009, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Software Engineering, University of Western Australia, 35 Stirling Highway, Perth 6009, Australia","institution_ids":["https://openalex.org/I177877127"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5084633048"],"corresponding_institution_ids":["https://openalex.org/I134085113","https://openalex.org/I177877127"],"apc_list":{"value":1400,"currency":"CHF","value_usd":1515},"apc_paid":{"value":1400,"currency":"CHF","value_usd":1515},"fwci":2.6085,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.91372356,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":98},"biblio":{"volume":"16","issue":"11","first_page":"937","last_page":"937"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3734999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3734999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.10329999774694443,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.1023000031709671,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7906000018119812},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.7861999869346619},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.5953999757766724},{"id":"https://openalex.org/keywords/scope","display_name":"Scope (computer science)","score":0.47350001335144043},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4629000127315521},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.44690001010894775}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7906000018119812},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7861999869346619},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5953999757766724},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5857999920845032},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.567799985408783},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5220000147819519},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.47350001335144043},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4629000127315521},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.44690001010894775},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4162999987602234},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3939000070095062},{"id":"https://openalex.org/C2781256819","wikidata":"https://www.wikidata.org/wiki/Q16828835","display_name":"Antecedent (behavioral psychology)","level":2,"score":0.3513999879360199},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.34700000286102295},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.32749998569488525},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.29409998655319214},{"id":"https://openalex.org/C2908998935","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Visual Hallucination","level":2,"score":0.2915000021457672},{"id":"https://openalex.org/C2911011789","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Hallucinating","level":2,"score":0.25940001010894775}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.3390/info16110937","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info16110937","pdf_url":"https://www.mdpi.com/2078-2489/16/11/937/pdf?version=1761653353","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},{"id":"pmh:oai:pure.atira.dk:publications/af5ba084-8ac1-4f75-8142-039843910fa6","is_oa":true,"landing_page_url":"https://admin.research-repository.uwa.edu.au/en/publications/af5ba084-8ac1-4f75-8142-039843910fa6","pdf_url":null,"source":{"id":"https://openalex.org/S4306402492","display_name":"UWA Profiles and Research Repository (UWA)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I177877127","host_organization_name":"The University of Western Australia","host_organization_lineage":["https://openalex.org/I177877127"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Rahman, A B M A, Anwar, S, Usman, M, Ahmad, I & Mian, A 2025, 'DefAn : Definitive Answer Dataset for LLM Hallucination Evaluation', Information (Switzerland), vol. 16, no. 11, 937. https://doi.org/10.3390/info16110937","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:pure.atira.dk:openaire/af5ba084-8ac1-4f75-8142-039843910fa6","is_oa":true,"landing_page_url":"https://research-repository.uwa.edu.au/en/publications/af5ba084-8ac1-4f75-8142-039843910fa6","pdf_url":null,"source":{"id":"https://openalex.org/S4306402492","display_name":"UWA Profiles and Research Repository (UWA)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I177877127","host_organization_name":"The University of Western Australia","host_organization_lineage":["https://openalex.org/I177877127"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Rahman, A B M A, Anwar, S, Usman, M, Ahmad, I & Mian, A 2025, 'DefAn : Definitive Answer Dataset for LLM Hallucination Evaluation', Information (Switzerland), vol. 16, no. 11, 937. https://doi.org/10.3390/info16110937","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:doaj.org/article:f18cd542e7ea40b9b367fd77f12e0b91","is_oa":true,"landing_page_url":"https://doaj.org/article/f18cd542e7ea40b9b367fd77f12e0b91","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Information, Vol 16, Iss 11, p 937 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3390/info16110937","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info16110937","pdf_url":"https://www.mdpi.com/2078-2489/16/11/937/pdf?version=1761653353","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322323","display_name":"King Fahd University of Petroleum and Minerals","ror":"https://ror.org/03yez3163"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415683709.pdf","grobid_xml":"https://content.openalex.org/works/W4415683709.grobid-xml"},"referenced_works_count":14,"referenced_works":["https://openalex.org/W3201174429","https://openalex.org/W4309674289","https://openalex.org/W4389518784","https://openalex.org/W4389518833","https://openalex.org/W4389523799","https://openalex.org/W4398757454","https://openalex.org/W4400113208","https://openalex.org/W4400688542","https://openalex.org/W4402671735","https://openalex.org/W4402671783","https://openalex.org/W4403223051","https://openalex.org/W4411403346","https://openalex.org/W4411630296","https://openalex.org/W4412887922"],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"represent":[4],"a":[5,93,140,260],"major":[6],"step":[7],"in":[8,15,66,88,107,203,211],"AI":[9],"development":[10],"and":[11,33,56,68,70,120,135,139,172,188,207,222,250,257],"are":[12,20,64,76,114],"increasingly":[13],"used":[14],"daily":[16],"applications.":[17],"However,":[18],"they":[19],"prone":[21],"to":[22,51,116,182,190,201,209,220,224],"hallucinations,":[23],"generating":[24],"claims":[25],"that":[26,175,230],"contradict":[27],"established":[28],"facts,":[29],"deviating":[30],"from":[31,180,218],"prompts,":[32],"producing":[34],"inconsistent":[35],"responses":[36],"when":[37,235],"the":[38,52,80,185,193,204,212],"same":[39],"prompt":[40],"is":[41,48,125],"presented":[42],"multiple":[43],"times.":[44],"Addressing":[45],"these":[46],"issues":[47],"challenging":[49],"due":[50],"lack":[53],"of":[54,83,98],"comprehensive":[55,94,261],"easily":[57],"assessable":[58],"benchmark":[59,95,262],"datasets.":[60],"Most":[61],"existing":[62],"datasets":[63],"limited":[65],"scale":[67],"scope":[69],"rely":[71],"on":[72,184,192],"multiple-choice":[73],"questions,":[74],"which":[75],"insufficient":[77],"for":[78,133,143,237,263],"evaluating":[79,264],"generative":[81],"capabilities":[82],"LLMs.":[84,146],"To":[85],"assess":[86],"hallucination":[87,178],"LLMs,":[89],"this":[90],"paper":[91],"introduces":[92],"dataset":[96,124,187,206,253],"consisting":[97],"over":[99],"20,000":[100],"unique":[101],"prompts":[102,106,113],"(more":[103],"than":[104],"75,000":[105],"total)":[108],"across":[109],"eight":[110],"domains.":[111],"These":[112],"designed":[115],"elicit":[117],"definitive,":[118],"concise,":[119],"informative":[121],"answers.":[122],"The":[123],"divided":[126],"into":[127],"two":[128],"segments:":[129],"one":[130],"publicly":[131],"available":[132],"testing":[134],"assessing":[136],"LLM":[137,231,265],"performance,":[138],"hidden":[141,194,213],"segment":[142],"benchmarking":[144],"various":[145],"In":[147],"our":[148],"experiments,":[149],"we":[150],"tested":[151],"nine":[152],"State-of-The-Art":[153],"(SoTA)":[154],"models,":[155],"GPT-4o,":[156],"GPT-3.5,":[157],"LLama":[158,161],"2":[159],"7B,":[160,170],"3":[162],"8B,":[163],"Gemini":[164],"1.0":[165],"Pro,":[166],"Mixtral":[167],"8x7B,":[168],"Zephyr":[169],"Deepseek-r1-7b,":[171],"Qwen2.5-14B,":[173],"revealing":[174],"overall":[176],"factual":[177],"ranges":[179,199,217],"48%":[181],"82%":[183],"public":[186,205],"31%":[189],"76%":[191],"benchmark.":[195],"Prompt":[196],"Misalignment":[197],"Hallucination":[198],"up":[200,208],"95%":[202],"94%":[210],"counterpart.":[214],"Average":[215],"consistency":[216],"21%":[219],"61%":[221],"44%":[223],"63%,":[225],"respectively.":[226],"Domain-wise":[227],"analysis":[228],"reveals":[229],"performance":[232],"significantly":[233],"deteriorates":[234],"asked":[236],"specific":[238],"numeric":[239],"information,":[240],"whereas":[241],"it":[242],"performs":[243],"moderately":[244],"with":[245],"queries":[246],"involving":[247],"persons,":[248],"locations,":[249],"dates.":[251],"Our":[252],"demonstrates":[254],"its":[255],"efficacy":[256],"serves":[258],"as":[259],"performance.":[266]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-30T00:00:00"}