{"id":"https://openalex.org/W4410343193","doi":"https://doi.org/10.1038/s41746-025-01670-7","title":"A framework to assess clinical safety and hallucination rates of LLMs for medical text summarisation","display_name":"A framework to assess clinical safety and hallucination rates of LLMs for medical text summarisation","publication_year":2025,"publication_date":"2025-05-13","ids":{"openalex":"https://openalex.org/W4410343193","doi":"https://doi.org/10.1038/s41746-025-01670-7","pmid":"https://pubmed.ncbi.nlm.nih.gov/40360677"},"language":"en","primary_location":{"id":"doi:10.1038/s41746-025-01670-7","is_oa":true,"landing_page_url":"https://doi.org/10.1038/s41746-025-01670-7","pdf_url":"https://www.nature.com/articles/s41746-025-01670-7.pdf","source":{"id":"https://openalex.org/S4210195431","display_name":"npj Digital Medicine","issn_l":"2398-6352","issn":["2398-6352"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319908","host_organization_name":"Nature Portfolio","host_organization_lineage":["https://openalex.org/P4310319908","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Nature Portfolio","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"npj Digital Medicine","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.nature.com/articles/s41746-025-01670-7.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040599352","display_name":"Elham Asgari","orcid":"https://orcid.org/0000-0002-5692-0784"},"institutions":[{"id":"https://openalex.org/I200166805","display_name":"Guy's and St Thomas' NHS Foundation Trust","ror":"https://ror.org/00j161312","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I200166805"]},{"id":"https://openalex.org/I4210131620","display_name":"Newham University Hospital","ror":"https://ror.org/045r28721","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I225661044","https://openalex.org/I4210131620"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Elham Asgari","raw_affiliation_strings":["Guy's and St Thomas NHS Trust, London, UK. asgelham@gmail.com","Tortus AI, London, UK. asgelham@gmail.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Guy's and St Thomas NHS Trust, London, UK. asgelham@gmail.com","institution_ids":["https://openalex.org/I200166805"]},{"raw_affiliation_string":"Tortus AI, London, UK. asgelham@gmail.com","institution_ids":["https://openalex.org/I4210131620"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023808275","display_name":"Nina Monta\u00f1a-Brown","orcid":"https://orcid.org/0000-0001-5685-971X"},"institutions":[{"id":"https://openalex.org/I3019271933","display_name":"Nicolaus Copernicus University","ror":"https://ror.org/0102mm775","country_code":"PL","type":"education","lineage":["https://openalex.org/I3019271933"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Nina Monta\u00f1a-Brown","raw_affiliation_strings":["Tortus AI, London, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tortus AI, London, UK","institution_ids":["https://openalex.org/I3019271933"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055765010","display_name":"Magda Dubois","orcid":"https://orcid.org/0000-0002-5396-1855"},"institutions":[{"id":"https://openalex.org/I3019271933","display_name":"Nicolaus Copernicus University","ror":"https://ror.org/0102mm775","country_code":"PL","type":"education","lineage":["https://openalex.org/I3019271933"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Magda Dubois","raw_affiliation_strings":["Tortus AI, London, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tortus AI, London, UK","institution_ids":["https://openalex.org/I3019271933"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079420099","display_name":"Saleh Khalil","orcid":null},"institutions":[{"id":"https://openalex.org/I3019271933","display_name":"Nicolaus Copernicus University","ror":"https://ror.org/0102mm775","country_code":"PL","type":"education","lineage":["https://openalex.org/I3019271933"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Saleh Khalil","raw_affiliation_strings":["Tortus AI, London, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tortus AI, London, UK","institution_ids":["https://openalex.org/I3019271933"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093539167","display_name":"Jasmine Balloch","orcid":"https://orcid.org/0009-0004-2322-1356"},"institutions":[{"id":"https://openalex.org/I3019271933","display_name":"Nicolaus Copernicus University","ror":"https://ror.org/0102mm775","country_code":"PL","type":"education","lineage":["https://openalex.org/I3019271933"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Jasmine Balloch","raw_affiliation_strings":["Tortus AI, London, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tortus AI, London, UK","institution_ids":["https://openalex.org/I3019271933"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024160492","display_name":"Joshua Au Yeung","orcid":"https://orcid.org/0000-0002-9428-2146"},"institutions":[{"id":"https://openalex.org/I3019271933","display_name":"Nicolaus Copernicus University","ror":"https://ror.org/0102mm775","country_code":"PL","type":"education","lineage":["https://openalex.org/I3019271933"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Joshua Au Yeung","raw_affiliation_strings":["Tortus AI, London, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tortus AI, London, UK","institution_ids":["https://openalex.org/I3019271933"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5071158374","display_name":"Dominic Pimenta","orcid":"https://orcid.org/0000-0002-3179-7249"},"institutions":[{"id":"https://openalex.org/I3019271933","display_name":"Nicolaus Copernicus University","ror":"https://ror.org/0102mm775","country_code":"PL","type":"education","lineage":["https://openalex.org/I3019271933"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Dominic Pimenta","raw_affiliation_strings":["Tortus AI, London, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tortus AI, London, UK","institution_ids":["https://openalex.org/I3019271933"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5040599352"],"corresponding_institution_ids":["https://openalex.org/I200166805","https://openalex.org/I4210131620"],"apc_list":{"value":3590,"currency":"USD","value_usd":3590},"apc_paid":{"value":3590,"currency":"USD","value_usd":3590},"fwci":108.3896,"has_fulltext":true,"cited_by_count":268,"citation_normalized_percentile":{"value":0.99994945,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"8","issue":"1","first_page":"274","last_page":"274"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9865000247955322,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.4515969157218933},{"id":"https://openalex.org/keywords/psychiatry","display_name":"Psychiatry","score":0.3222353756427765}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4515969157218933},{"id":"https://openalex.org/C118552586","wikidata":"https://www.wikidata.org/wiki/Q7867","display_name":"Psychiatry","level":1,"score":0.3222353756427765}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1038/s41746-025-01670-7","is_oa":true,"landing_page_url":"https://doi.org/10.1038/s41746-025-01670-7","pdf_url":"https://www.nature.com/articles/s41746-025-01670-7.pdf","source":{"id":"https://openalex.org/S4210195431","display_name":"npj Digital Medicine","issn_l":"2398-6352","issn":["2398-6352"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319908","host_organization_name":"Nature Portfolio","host_organization_lineage":["https://openalex.org/P4310319908","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Nature Portfolio","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"npj Digital Medicine","raw_type":"journal-article"},{"id":"pmid:40360677","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40360677","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"NPJ digital medicine","raw_type":null},{"id":"pmh:oai:doaj.org/article:39fe66968aa1429989174939cceefe4f","is_oa":true,"landing_page_url":"https://doaj.org/article/39fe66968aa1429989174939cceefe4f","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"npj Digital Medicine, Vol 8, Iss 1, Pp 1-15 (2025)","raw_type":"article"},{"id":"pmh:oai:pubmedcentral.nih.gov:12075489","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/12075489","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"NPJ Digit Med","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1038/s41746-025-01670-7","is_oa":true,"landing_page_url":"https://doi.org/10.1038/s41746-025-01670-7","pdf_url":"https://www.nature.com/articles/s41746-025-01670-7.pdf","source":{"id":"https://openalex.org/S4210195431","display_name":"npj Digital Medicine","issn_l":"2398-6352","issn":["2398-6352"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319908","host_organization_name":"Nature Portfolio","host_organization_lineage":["https://openalex.org/P4310319908","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Nature Portfolio","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"npj Digital Medicine","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4410343193.pdf","grobid_xml":"https://content.openalex.org/works/W4410343193.grobid-xml"},"referenced_works_count":53,"referenced_works":["https://openalex.org/W1994256263","https://openalex.org/W2101105183","https://openalex.org/W2114014706","https://openalex.org/W2152772232","https://openalex.org/W2166750785","https://openalex.org/W2936695845","https://openalex.org/W2941604507","https://openalex.org/W2963926728","https://openalex.org/W2995225687","https://openalex.org/W3027879771","https://openalex.org/W3116079511","https://openalex.org/W3133702157","https://openalex.org/W3186721116","https://openalex.org/W4221143046","https://openalex.org/W4285242550","https://openalex.org/W4286905034","https://openalex.org/W4287213686","https://openalex.org/W4309674289","https://openalex.org/W4313197536","https://openalex.org/W4318350718","https://openalex.org/W4319301505","https://openalex.org/W4323350039","https://openalex.org/W4378770578","https://openalex.org/W4380353763","https://openalex.org/W4382930233","https://openalex.org/W4385436582","https://openalex.org/W4386120650","https://openalex.org/W4386528723","https://openalex.org/W4386576626","https://openalex.org/W4386942825","https://openalex.org/W4387500346","https://openalex.org/W4388444944","https://openalex.org/W4388585881","https://openalex.org/W4389072658","https://openalex.org/W4390602555","https://openalex.org/W4391158659","https://openalex.org/W4391494845","https://openalex.org/W4391766565","https://openalex.org/W4391995913","https://openalex.org/W4392077089","https://openalex.org/W4392193048","https://openalex.org/W4392668324","https://openalex.org/W4392711449","https://openalex.org/W4393971165","https://openalex.org/W4394759276","https://openalex.org/W4395049441","https://openalex.org/W4395483680","https://openalex.org/W4399803256","https://openalex.org/W4399911714","https://openalex.org/W4402348202","https://openalex.org/W4402909345","https://openalex.org/W4404787535","https://openalex.org/W4404986534"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2931662336","https://openalex.org/W4220667126","https://openalex.org/W2077865380","https://openalex.org/W3006817050","https://openalex.org/W4401768695","https://openalex.org/W2765597752","https://openalex.org/W2134894512","https://openalex.org/W2083375246"],"abstract_inverted_index":{"Integrating":[0],"large":[1],"language":[2],"models":[3],"(LLMs)":[4],"into":[5],"healthcare":[6],"can":[7],"enhance":[8],"workflow":[9],"efficiency":[10],"and":[11,27,81,119,127],"patient":[12,42],"care":[13],"by":[14],"automating":[15],"tasks":[16],"such":[17],"as":[18],"summarising":[19],"consultations.":[20],"However,":[21],"the":[22,77,141],"fidelity":[23],"between":[24],"LLM":[25,55,66],"outputs":[26],"ground":[28],"truth":[29],"information":[30],"is":[31],"vital":[32],"to":[33,39,75,88],"prevent":[34],"miscommunication":[35],"that":[36],"could":[37],"lead":[38],"compromise":[40],"in":[41,64],"safety.":[43],"We":[44,113],"propose":[45],"a":[46,71,83,115,120],"framework":[47,74],"comprising":[48],"(1)":[49],"an":[50,58],"error":[51,94],"taxonomy":[52],"for":[53,61,104,144],"classifying":[54],"outputs,":[56],"(2)":[57],"experimental":[59,100],"structure":[60],"iterative":[62],"comparisons":[63],"our":[65],"document":[67],"generation":[68],"pipeline,":[69],"(3)":[70],"clinical":[72,93,105,146],"safety":[73],"evaluate":[76],"harms":[78],"of":[79,109],"errors,":[80],"(4)":[82],"graphical":[84],"user":[85],"interface,":[86],"CREOLA,":[87],"facilitate":[89],"these":[90],"processes.":[91],"Our":[92],"metrics":[95],"were":[96],"derived":[97],"from":[98],"18":[99],"configurations":[101],"involving":[102],"LLMs":[103],"note":[106],"generation,":[107],"consisting":[108],"12,999":[110],"clinician-annotated":[111],"sentences.":[112],"observed":[114],"1.47%":[116],"hallucination":[117],"rate":[118],"3.45%":[121],"omission":[122],"rate.":[123],"By":[124],"refining":[125],"prompts":[126],"workflows,":[128],"we":[129],"successfully":[130],"reduced":[131],"major":[132],"errors":[133],"below":[134],"previously":[135],"reported":[136],"human":[137],"note-taking":[138],"rates,":[139],"highlighting":[140],"framework's":[142],"potential":[143],"safer":[145],"documentation.":[147]},"counts_by_year":[{"year":2026,"cited_by_count":190},{"year":2025,"cited_by_count":75},{"year":2024,"cited_by_count":3}],"updated_date":"2026-07-30T17:31:21.811387","created_date":"2025-10-10T00:00:00"}