{"id":"https://openalex.org/W4414891332","doi":"https://doi.org/10.1038/s41746-025-01963-x","title":"The evaluation illusion of large language models in medicine","display_name":"The evaluation illusion of large language models in medicine","publication_year":2025,"publication_date":"2025-10-07","ids":{"openalex":"https://openalex.org/W4414891332","doi":"https://doi.org/10.1038/s41746-025-01963-x","pmid":"https://pubmed.ncbi.nlm.nih.gov/41057566"},"language":"en","primary_location":{"id":"doi:10.1038/s41746-025-01963-x","is_oa":true,"landing_page_url":"https://doi.org/10.1038/s41746-025-01963-x","pdf_url":"https://www.nature.com/articles/s41746-025-01963-x.pdf","source":{"id":"https://openalex.org/S4210195431","display_name":"npj Digital Medicine","issn_l":"2398-6352","issn":["2398-6352"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319908","host_organization_name":"Nature Portfolio","host_organization_lineage":["https://openalex.org/P4310319908","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Nature Portfolio","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"npj Digital Medicine","raw_type":"journal-article"},"type":"letter","indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.nature.com/articles/s41746-025-01963-x.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5115858431","display_name":"Monica Agrawal","orcid":null},"institutions":[{"id":"https://openalex.org/I170897317","display_name":"Duke University","ror":"https://ror.org/00py81415","country_code":"US","type":"education","lineage":["https://openalex.org/I170897317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Monica Agrawal","raw_affiliation_strings":["Department of Biostatistics and Biomedical Informatics, Duke University, Durham, NC, USA","Department of Computer Science, Duke University, Durham, NC, USA"],"affiliations":[{"raw_affiliation_string":"Department of Biostatistics and Biomedical Informatics, Duke University, Durham, NC, USA","institution_ids":["https://openalex.org/I170897317"]},{"raw_affiliation_string":"Department of Computer Science, Duke University, Durham, NC, USA","institution_ids":["https://openalex.org/I170897317"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081135036","display_name":"Irene Y. Chen","orcid":"https://orcid.org/0000-0001-5202-2111"},"institutions":[{"id":"https://openalex.org/I180670191","display_name":"University of California, San Francisco","ror":"https://ror.org/043mz5j54","country_code":"US","type":"education","lineage":["https://openalex.org/I180670191"]},{"id":"https://openalex.org/I76766440","display_name":"University of San Francisco","ror":"https://ror.org/029m7xn54","country_code":"US","type":"education","lineage":["https://openalex.org/I76766440"]},{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Irene Y. Chen","raw_affiliation_strings":["Computational Precision Health, University of California, Berkeley, Berkeley, CA, USA. iychen@berkeley.edu","Department of Electrical Engineering and Computer Science, University of California, Berkeley, Berkeley, CA, USA. iychen@berkeley.edu","University of California, San Francisco, San Francisco, CA, USA. iychen@berkeley.edu","Department of Electrical Engineering and Computer Science, University of California, Berkeley, Berkeley, CA, USA","University of California, San Francisco, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Computational Precision Health, University of California, Berkeley, Berkeley, CA, USA. iychen@berkeley.edu","institution_ids":["https://openalex.org/I95457486"]},{"raw_affiliation_string":"Department of Electrical Engineering and Computer Science, University of California, Berkeley, Berkeley, CA, USA. iychen@berkeley.edu","institution_ids":["https://openalex.org/I95457486"]},{"raw_affiliation_string":"University of California, San Francisco, San Francisco, CA, USA. iychen@berkeley.edu","institution_ids":["https://openalex.org/I180670191","https://openalex.org/I76766440","https://openalex.org/I95457486"]},{"raw_affiliation_string":"Department of Electrical Engineering and Computer Science, University of California, Berkeley, Berkeley, CA, USA","institution_ids":["https://openalex.org/I95457486"]},{"raw_affiliation_string":"University of California, San Francisco, San Francisco, CA, USA","institution_ids":["https://openalex.org/I180670191"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119876625","display_name":"Freya Gulamali","orcid":null},"institutions":[{"id":"https://openalex.org/I170897317","display_name":"Duke University","ror":"https://ror.org/00py81415","country_code":"US","type":"education","lineage":["https://openalex.org/I170897317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Freya Gulamali","raw_affiliation_strings":["Department of Computer Science, Duke University, Durham, NC, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Duke University, Durham, NC, USA","institution_ids":["https://openalex.org/I170897317"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074121216","display_name":"Shalmali Joshi","orcid":"https://orcid.org/0009-0002-6594-4902"},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shalmali Joshi","raw_affiliation_strings":["Department of Biomedical Informatics, Columbia University, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"Department of Biomedical Informatics, Columbia University, New York, NY, USA","institution_ids":["https://openalex.org/I78577930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5081135036"],"corresponding_institution_ids":["https://openalex.org/I180670191","https://openalex.org/I76766440","https://openalex.org/I95457486"],"apc_list":{"value":3060,"currency":"USD","value_usd":3060},"apc_paid":{"value":3060,"currency":"USD","value_usd":3060},"fwci":24.9585,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.99408863,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"8","issue":"1","first_page":"600","last_page":"600"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9926000237464905,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9912999868392944,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transparency","display_name":"Transparency (behavior)","score":0.506600022315979},{"id":"https://openalex.org/keywords/illusion","display_name":"Illusion","score":0.4708000123500824},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4603999853134155},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4410000145435333},{"id":"https://openalex.org/keywords/limit","display_name":"Limit (mathematics)","score":0.37770000100135803},{"id":"https://openalex.org/keywords/language-understanding","display_name":"Language understanding","score":0.3531000018119812}],"concepts":[{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.506600022315979},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4959999918937683},{"id":"https://openalex.org/C184047640","wikidata":"https://www.wikidata.org/wiki/Q182593","display_name":"Illusion","level":2,"score":0.4708000123500824},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4603999853134155},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4526999890804291},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4410000145435333},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.41589999198913574},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3801000118255615},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.37770000100135803},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.35910001397132874},{"id":"https://openalex.org/C2983448237","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Language understanding","level":2,"score":0.3531000018119812},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.31290000677108765},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.3084999918937683},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.29660001397132874},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.27720001339912415},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.2727999985218048},{"id":"https://openalex.org/C2779313563","wikidata":"https://www.wikidata.org/wiki/Q17072565","display_name":"On Language","level":2,"score":0.2623000144958496},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.2556000053882599}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1038/s41746-025-01963-x","is_oa":true,"landing_page_url":"https://doi.org/10.1038/s41746-025-01963-x","pdf_url":"https://www.nature.com/articles/s41746-025-01963-x.pdf","source":{"id":"https://openalex.org/S4210195431","display_name":"npj Digital Medicine","issn_l":"2398-6352","issn":["2398-6352"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319908","host_organization_name":"Nature Portfolio","host_organization_lineage":["https://openalex.org/P4310319908","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Nature Portfolio","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"npj Digital Medicine","raw_type":"journal-article"},{"id":"pmid:41057566","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/41057566","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"NPJ digital medicine","raw_type":null},{"id":"pmh:oai:doaj.org/article:662c171449f34b68b600c3eae95918b7","is_oa":true,"landing_page_url":"https://doaj.org/article/662c171449f34b68b600c3eae95918b7","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"npj Digital Medicine, Vol 8, Iss 1, Pp 1-4 (2025)","raw_type":"article"},{"id":"pmh:oai:europepmc.org:11312439","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/12504413","pdf_url":null,"source":{"id":"https://openalex.org/S4306400806","display_name":"Europe PMC (PubMed Central)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1303153112","host_organization_name":"European Bioinformatics Institute","host_organization_lineage":["https://openalex.org/I1303153112"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"},{"id":"pmh:oai:pubmedcentral.nih.gov:12504413","is_oa":true,"landing_page_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC12504413/","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"NPJ Digit Med","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1038/s41746-025-01963-x","is_oa":true,"landing_page_url":"https://doi.org/10.1038/s41746-025-01963-x","pdf_url":"https://www.nature.com/articles/s41746-025-01963-x.pdf","source":{"id":"https://openalex.org/S4210195431","display_name":"npj Digital Medicine","issn_l":"2398-6352","issn":["2398-6352"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319908","host_organization_name":"Nature Portfolio","host_organization_lineage":["https://openalex.org/P4310319908","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Nature Portfolio","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"npj Digital Medicine","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4414891332.pdf","grobid_xml":"https://content.openalex.org/works/W4414891332.grobid-xml"},"referenced_works_count":30,"referenced_works":["https://openalex.org/W2101105183","https://openalex.org/W2143514833","https://openalex.org/W2557060223","https://openalex.org/W2592774571","https://openalex.org/W2888120268","https://openalex.org/W3087893815","https://openalex.org/W3159623667","https://openalex.org/W3176894732","https://openalex.org/W4285242550","https://openalex.org/W4385571331","https://openalex.org/W4386566731","https://openalex.org/W4389917881","https://openalex.org/W4392193048","https://openalex.org/W4392986561","https://openalex.org/W4394674889","https://openalex.org/W4394807113","https://openalex.org/W4396832118","https://openalex.org/W4399994866","https://openalex.org/W4400921803","https://openalex.org/W4402667136","https://openalex.org/W4402909345","https://openalex.org/W4403420208","https://openalex.org/W4404782815","https://openalex.org/W4405225674","https://openalex.org/W4405995927","https://openalex.org/W4406137005","https://openalex.org/W4406740982","https://openalex.org/W4409020930","https://openalex.org/W4413941679","https://openalex.org/W6891805778"],"related_works":[],"abstract_inverted_index":{"While":[0],"large":[1,17],"language":[2,18],"models":[3,19],"(LLMs)":[4],"hold":[5],"promise":[6],"for":[7,56],"transforming":[8],"clinical":[9],"healthcare,":[10],"current":[11],"comparisons":[12],"and":[13,40,49,60,66],"benchmark":[14],"evaluations":[15,59],"of":[16,37,46],"in":[20],"medicine":[21],"often":[22],"fail":[23],"to":[24],"capture":[25],"real-world":[26],"efficacy.":[27],"Specifically,":[28],"we":[29,54],"highlight":[30],"how":[31],"key":[32],"discrepancies":[33],"arising":[34],"from":[35],"choices":[36],"data,":[38],"tasks,":[39],"metrics":[41],"can":[42],"limit":[43],"meaningful":[44],"assessment":[45],"translational":[47],"impact":[48],"cause":[50],"misleading":[51],"conclusions.":[52],"Therefore,":[53],"advocate":[55],"rigorous,":[57],"context-aware":[58],"experimental":[61],"transparency":[62],"across":[63],"both":[64],"research":[65],"deployment.":[67]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":5}],"updated_date":"2026-03-28T08:17:26.163206","created_date":"2025-10-10T00:00:00"}