{"id":"https://openalex.org/W7140318072","doi":"https://doi.org/10.48550/arxiv.2603.22344","title":"Errors in AI-Assisted Retrieval of Medical Literature: A Comparative Study","display_name":"Errors in AI-Assisted Retrieval of Medical Literature: A Comparative Study","publication_year":2026,"publication_date":"2026-03-21","ids":{"openalex":"https://openalex.org/W7140318072","doi":"https://doi.org/10.48550/arxiv.2603.22344"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2603.22344","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.22344","pdf_url":"https://arxiv.org/pdf/2603.22344","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2603.22344","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5130641245","display_name":"Jenny Gao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gao, Jenny","raw_affiliation_strings":["College of Arts and Science, New York University, New York, NY"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Arts and Science, New York University, New York, NY","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130571126","display_name":"Yongfeng Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhang, Yongfeng","raw_affiliation_strings":["Department of Computer Sciences, School of Arts & Sciences, Rutgers University, Piscataway, NJ"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Sciences, School of Arts & Sciences, Rutgers University, Piscataway, NJ","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039848435","display_name":"Mary Lenora Disis","orcid":"https://orcid.org/0000-0001-7653-4648","observed_orcids":["https://orcid.org/0000-0001-7653-4648"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Disis, Mary L","raw_affiliation_strings":["UW Medicine Cancer Vaccine Institute University of Washington, Seattle, WA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UW Medicine Cancer Vaccine Institute University of Washington, Seattle, WA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046441324","display_name":"Lanjing Zhang","orcid":"https://orcid.org/0000-0001-5436-887X","observed_orcids":["https://orcid.org/0000-0001-5436-887X"]},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]},{"id":"https://openalex.org/I4210094960","display_name":"University Medical Center of Princeton","ror":"https://ror.org/00mbhee72","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210094960"]},{"id":"https://openalex.org/I4390039325","display_name":"Rutgers Cancer Institute","ror":"https://ror.org/0060x3y55","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I102322142","https://openalex.org/I4390039302","https://openalex.org/I4390039325"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhang, Lanjing","raw_affiliation_strings":["Department of Chemical Biology, Ernest Mario School of Pharmacy, Rutgers University, Piscataway, NJ","Department of Pathology, Princeton Medical Center, Plainsboro, NJ","Rutgers Cancer Institute, New Brunswick, NJ"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Chemical Biology, Ernest Mario School of Pharmacy, Rutgers University, Piscataway, NJ","institution_ids":["https://openalex.org/I102322142"]},{"raw_affiliation_string":"Department of Pathology, Princeton Medical Center, Plainsboro, NJ","institution_ids":["https://openalex.org/I4210094960"]},{"raw_affiliation_string":"Rutgers Cancer Institute, New Brunswick, NJ","institution_ids":["https://openalex.org/I4390039325"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.7383000254631042,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.7383000254631042,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.2012999951839447,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.014999999664723873,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9330000281333923},{"id":"https://openalex.org/keywords/retrieval-errors","display_name":"retrieval errors","score":0.6679999828338623},{"id":"https://openalex.org/keywords/reference-retrieval","display_name":"reference retrieval","score":0.7170000076293945},{"id":"https://openalex.org/keywords/medical-literature","display_name":"medical literature","score":0.47200000286102295},{"id":"https://openalex.org/keywords/chatgpt","display_name":"ChatGPT","score":0.8420000076293945},{"id":"https://openalex.org/keywords/google-gemini","display_name":"Google Gemini","score":0.8970000147819519},{"id":"https://openalex.org/keywords/perplexity-ai","display_name":"Perplexity AI","score":0.9049999713897705},{"id":"https://openalex.org/keywords/deepseek","display_name":"DeepSeek","score":0.8920000195503235}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.597599983215332},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5759999752044678},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.4131999909877777},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.39730000495910645},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.38589999079704285},{"id":"https://openalex.org/C151956035","wikidata":"https://www.wikidata.org/wiki/Q1132755","display_name":"Logistic regression","level":2,"score":0.35280001163482666},{"id":"https://openalex.org/C117312493","wikidata":"https://www.wikidata.org/wiki/Q2035437","display_name":"Multivariable calculus","level":2,"score":0.34940001368522644},{"id":"https://openalex.org/C2779231881","wikidata":"https://www.wikidata.org/wiki/Q5977147","display_name":"Medical literature","level":2,"score":0.3215000033378601},{"id":"https://openalex.org/C145642194","wikidata":"https://www.wikidata.org/wiki/Q870895","display_name":"Health informatics","level":3,"score":0.3190999925136566},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2964000105857849},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.28850001096725464},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2775000035762787},{"id":"https://openalex.org/C513874922","wikidata":"https://www.wikidata.org/wiki/Q212805","display_name":"Digital library","level":3,"score":0.27709999680519104}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2603.22344","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.22344","pdf_url":"https://arxiv.org/pdf/2603.22344","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2603.22344","doi":"https://doi.org/10.48550/arxiv.2603.22344","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.22344","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2603.22344","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.22344","pdf_url":"https://arxiv.org/pdf/2603.22344","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G9000911452","display_name":"SCH: Screening and confirmatory machine learning for explainable modeling of non-cancer deaths in cancer patients","funder_award_id":"R37CA277812","funder_id":"https://openalex.org/F4320337351","funder_display_name":"National Cancer Institute"}],"funders":[{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320337351","display_name":"National Cancer Institute","ror":"https://ror.org/040gcmg81"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7140318072.pdf","grobid_xml":"https://content.openalex.org/works/W7140318072.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4410458608","https://openalex.org/W4416386321","https://openalex.org/W7165197584","https://openalex.org/W4391590636","https://openalex.org/W7208730388","https://openalex.org/W7154566100","https://openalex.org/W7151881841","https://openalex.org/W7220670553","https://openalex.org/W4410939659","https://openalex.org/W4409505176"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"assisted":[4],"literature":[5,264],"retrieval":[6,27,40,236],"may":[7],"lead":[8],"to":[9,73,129,137],"erroneous":[10],"references,":[11],"but":[12],"these":[13],"errors":[14,24],"have":[15],"not":[16],"been":[17],"rigorously":[18],"quantified.":[19],"Therefore,":[20],"we":[21],"quantitatively":[22],"assess":[23],"in":[25,171,235,251],"reference":[26,140],"of":[28,82,92,104,119,143,150,230],"widely":[29],"used":[30,128],"free-version":[31],"LLM":[32,133,153,210,242],"platforms":[33,134,154,211,239,243],"and":[34,59,87,112,114,175,181,189,202,212,220,232,240,244],"identify":[35],"the":[36,83,144,151],"factors":[37],"associated":[38,216,247],"with":[39,162,194,217,248],"errors.":[41],"We":[42,225],"evaluated":[43],"2,000":[44],"references":[45,120,174],"retrieved":[46],"by":[47,186],"5":[48,152],"LLMs":[49,231],"(Grok-2,":[50],"ChatGPT":[51],"GPT-4.1,":[52],"Google":[53],"Gemini":[54],"Flash":[55],"2.5,":[56],"Perplexity":[57],"AI,":[58],"DeepSeek":[60],"GPT-4)":[61],"for":[62],"40":[63],"randomly-selected":[64],"original":[65],"articles":[66,197],"(10":[67],"per":[68],"journal)":[69],"published":[70],"Jan.":[71],"2024":[72],"July":[74],"2025":[75],"from":[76],"British":[77],"Medical":[78,85],"Journal":[79,81,91],"(BMJ),":[80],"American":[84],"Association,":[86],"The":[88,146,179],"New":[89],"England":[90],"Medicine":[93],"(NEJM).":[94],"Primary":[95],"outcomes":[96],"were":[97,184,214],"a":[98,163,168],"multimetric":[99],"score":[100,148,165,200,218],"ratio":[101,149,166],"combining":[102],"validity":[103],"digital":[105],"object":[106],"identifier,":[107],"PubMed":[108],"ID,":[109],"Google-Scholar":[110],"link,":[111],"relevance;":[113],"complete":[115,204,221],"miss":[116,205,222],"rate":[117],"(proportion":[118],"failing":[121],"all":[122],"applicable":[123],"metrics).":[124],"Multivariable":[125,207],"regression":[126],"was":[127,155],"examine":[130],"independent":[131],"associations.":[132],"completely":[135],"failed":[136],"retrieve":[138],"correct":[139,176],"data":[141,256],"47.8%":[142],"time.":[145],"average":[147],"0.29":[156],"(standard":[157],"deviation,":[158],"0.35;":[159],"range,":[160],"0-1.25),":[161],"higher":[164,169,203],"indicating":[167],"accuracy":[170,237],"retrieving":[172,252],"relevant":[173],"bibliographic":[177],"data.":[178],"highest":[180],"lowest":[182],"accuracies":[183],"achieved":[185],"Grok":[187],"(0.57)":[188],"Genimi":[190],"(0.11),":[191],"respectively.":[192,224],"Compared":[193],"BMJ,":[195],"NEJM":[196],"had":[198],"lower":[199],"ratios":[201,219],"rates.":[206],"analysis":[208],"shows":[209],"journals":[213,245],"independently":[215],"rate,":[223],"show":[226],"modest":[227],"overall":[228],"performance":[229,250],"significant":[233],"variability":[234],"across":[238],"journals.":[241],"are":[246],"LLM's":[249],"medical":[253],"literature.":[254],"Bibliographic":[255],"should":[257],"be":[258],"carefully":[259],"reviewed":[260],"when":[261],"using":[262],"LLM-assisted":[263],"retrieval.":[265]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2026-03-26T00:00:00"}