{"id":"https://openalex.org/W7135048965","doi":"https://doi.org/10.48550/arxiv.2603.09986","title":"Quantifying Hallucinations in Language Language Models on Medical Textbooks","display_name":"Quantifying Hallucinations in Language Language Models on Medical Textbooks","publication_year":2026,"publication_date":"2026-02-12","ids":{"openalex":"https://openalex.org/W7135048965","doi":"https://doi.org/10.48550/arxiv.2603.09986"},"language":"en","primary_location":{"id":"pmh:oai:pubmedcentral.nih.gov:13119327","is_oa":true,"landing_page_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC13119327/","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ArXiv","raw_type":"Text"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC13119327/","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119136078","display_name":"Brandon Colelough","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Colelough, Brandon C.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108892774","display_name":"Davis Bartels","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bartels, Davis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5079242885","display_name":"D. Demner-Fushman","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Demner-Fushman, Dina","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1257999986410141,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1257999986410141,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12488","display_name":"Mental Health via Writing","score":0.10209999978542328,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10094","display_name":"Epilepsy research and treatment","score":0.07609999924898148,"subfield":{"id":"https://openalex.org/subfields/2738","display_name":"Psychiatry and Mental health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hallucinating","display_name":"Hallucinating","score":0.9347000122070312},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.4242999851703644},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.39570000767707825},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.3255999982357025},{"id":"https://openalex.org/keywords/schizophrenia","display_name":"Schizophrenia (object-oriented programming)","score":0.3181999921798706},{"id":"https://openalex.org/keywords/language-proficiency","display_name":"Language proficiency","score":0.28279998898506165}],"concepts":[{"id":"https://openalex.org/C2911011789","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Hallucinating","level":2,"score":0.9347000122070312},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.6254000067710876},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.5475999712944031},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.4242999851703644},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.41819998621940613},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.39570000767707825},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.33640000224113464},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3255999982357025},{"id":"https://openalex.org/C2776412080","wikidata":"https://www.wikidata.org/wiki/Q7431605","display_name":"Schizophrenia (object-oriented programming)","level":2,"score":0.3181999921798706},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.30649998784065247},{"id":"https://openalex.org/C548259974","wikidata":"https://www.wikidata.org/wiki/Q569965","display_name":"Audiology","level":1,"score":0.3041999936103821},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.2948000133037567},{"id":"https://openalex.org/C70410870","wikidata":"https://www.wikidata.org/wiki/Q199906","display_name":"Clinical psychology","level":1,"score":0.2874999940395355},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2856999933719635},{"id":"https://openalex.org/C2778883600","wikidata":"https://www.wikidata.org/wiki/Q2390977","display_name":"Language proficiency","level":2,"score":0.28279998898506165},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.2727000117301941},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.27250000834465027},{"id":"https://openalex.org/C2908998935","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Visual Hallucination","level":2,"score":0.26420000195503235},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.26249998807907104}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:pubmedcentral.nih.gov:13119327","is_oa":true,"landing_page_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC13119327/","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ArXiv","raw_type":"Text"},{"id":"doi:10.48550/arxiv.2603.09986","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.09986","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:pubmedcentral.nih.gov:13119327","is_oa":true,"landing_page_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC13119327/","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ArXiv","raw_type":"Text"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.843158483505249}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Hallucinations,":[0],"the":[1,74,79,100,105,121,210],"tendency":[2],"for":[3,24,39,83,175,197],"large":[4,88,192],"language":[5,22,89,193],"models":[6,194],"to":[7,34,63,77,103,112,133,170,173],"provide":[8],"responses":[9,62,140],"with":[10,120,155],"factually":[11],"incorrect":[12],"and":[13,60,99,109,144,168,178,188,201,209],"unsupported":[14],"claims,":[15],"is":[16,206],"a":[17,47,84],"serious":[18],"problem":[19],"within":[20],"natural":[21],"processing":[23],"which":[25],"we":[26],"do":[27],"not":[28],"yet":[29],"have":[30],"an":[31],"effective":[32],"solution":[33],"mitigate":[35],"against.":[36],"Existing":[37],"benchmarks":[38],"medical":[40,64,93],"QA":[41,59,65,94],"rarely":[42],"evaluate":[43],"this":[44],"behavior":[45],"against":[46],"fixed":[48],"evidence":[49],"source.":[50],"We":[51,70,115],"ask":[52],"how":[53,61],"often":[54],"hallucinations":[55,82,108],"occur":[56],"on":[57],"textbook-grounded":[58],"prompts":[66],"vary":[67],"across":[68,149,185],"models.":[69],"conduct":[71],"two":[72],"experiments,":[73],"first":[75],"experiment":[76,102,118,147],"determine":[78,104],"prevalence":[80,106],"of":[81,107,128,138],"prominent":[85],"open":[86],"source":[87],"model":[90,113],"(LLaMA-70B-Instruct)":[91],"in":[92,117,126,146],"given":[95],"closed-source":[96],"zero-shot":[97],"prompts,":[98],"second":[101],"clinician":[110],"preference":[111],"responses.":[114],"observed,":[116],"one,":[119],"passages":[122],"provided,":[123],"LLaMA-70B-Instruct":[124],"hallucinated":[125],"19.7\\%":[127],"answers":[129],"(95\\%":[130],"CI":[131],"18.6":[132],"20.7)":[134],"even":[135],"though":[136],"98.8\\%":[137],"prompt":[139],"received":[141],"maximal":[142],"plausibility,":[143],"observed":[145],"two,":[148],"models,":[150],"lower":[151],"hallucination":[152],"rates":[153],"aligned":[154],"higher":[156],"usefulness":[157],"scores":[158],"($\u03c1=-0.71$,":[159],"$p=0.058$).":[160],"Clinicians":[161],"produced":[162],"high":[163],"agreement":[164],"(quadratic":[165],"weighted":[166],"$\u03ba=0.92$)":[167],"($\u03c4_b=0.06$":[169],"$0.18$,":[171],"$\u03ba=0.57$":[172],"$0.61$)":[174],"experiments":[176],"1":[177],"2":[179],"respectively.":[180],"Our":[181],"findings":[182],"indicate":[183],"that,":[184],"all":[186],"scales":[187],"architectures":[189],"tested,":[190],"current":[191],"remain":[195],"unfit":[196],"unsupervised":[198],"clinical":[199],"deployment,":[200],"that":[202],"human":[203],"expert":[204],"oversight":[205],"both":[207],"necessary":[208],"dominant":[211],"cost":[212],"driver.":[213]},"counts_by_year":[],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2026-03-13T00:00:00"}