{"id":"https://openalex.org/W7141560207","doi":"https://doi.org/10.48550/arxiv.2603.25196","title":"A Decade-Scale Benchmark Evaluating LLMs' Clinical Practice Guidelines Detection and Adherence in Multi-turn Conversations","display_name":"A Decade-Scale Benchmark Evaluating LLMs' Clinical Practice Guidelines Detection and Adherence in Multi-turn Conversations","publication_year":2026,"publication_date":"2026-03-26","ids":{"openalex":"https://openalex.org/W7141560207","doi":"https://doi.org/10.48550/arxiv.2603.25196"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.25196","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.25196","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.25196","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049600878","display_name":"Andong Tan","orcid":"https://orcid.org/0009-0008-4419-1837"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tan, Andong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130757586","display_name":"Shuyu Dai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dai, Shuyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070492669","display_name":"Jinglu Wang","orcid":"https://orcid.org/0000-0002-9786-8636"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jinglu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038686832","display_name":"Fengtao Zhou","orcid":"https://orcid.org/0000-0003-2518-7855"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Fengtao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130796589","display_name":"Yan Lu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Yan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130771686","display_name":"Xi Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104115576","display_name":"Yingcong Chen","orcid":"https://orcid.org/0009-0003-8989-2323"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yingcong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130804518","display_name":"Can Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Can","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130776257","display_name":"Shujie Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Shujie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5130750134","display_name":"Hao Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Hao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12664","display_name":"Clinical practice guidelines implementation","score":0.2833000123500824,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T12664","display_name":"Clinical practice guidelines implementation","score":0.2833000123500824,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10350","display_name":"Electronic Health Records Systems","score":0.257999986410141,"subfield":{"id":"https://openalex.org/subfields/3605","display_name":"Health Information Management"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.0885000005364418,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8529999852180481},{"id":"https://openalex.org/keywords/guideline","display_name":"Guideline","score":0.7278000116348267},{"id":"https://openalex.org/keywords/clinical-practice","display_name":"Clinical Practice","score":0.6359999775886536},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6212000250816345},{"id":"https://openalex.org/keywords/conversation","display_name":"Conversation","score":0.5497000217437744},{"id":"https://openalex.org/keywords/health-care","display_name":"Health care","score":0.5041999816894531},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.4480000138282776},{"id":"https://openalex.org/keywords/population","display_name":"Population","score":0.41830000281333923}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8529999852180481},{"id":"https://openalex.org/C2780182762","wikidata":"https://www.wikidata.org/wiki/Q1630279","display_name":"Guideline","level":2,"score":0.7278000116348267},{"id":"https://openalex.org/C2779974597","wikidata":"https://www.wikidata.org/wiki/Q28448986","display_name":"Clinical Practice","level":2,"score":0.6359999775886536},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6212000250816345},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.5497000217437744},{"id":"https://openalex.org/C160735492","wikidata":"https://www.wikidata.org/wiki/Q31207","display_name":"Health care","level":2,"score":0.5041999816894531},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.4560999870300293},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.4480000138282776},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.41830000281333923},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.40119999647140503},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.39910000562667847},{"id":"https://openalex.org/C12174686","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk assessment","level":2,"score":0.3181999921798706},{"id":"https://openalex.org/C2776035688","wikidata":"https://www.wikidata.org/wiki/Q1606558","display_name":"Affect (linguistics)","level":2,"score":0.31150001287460327},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.30000001192092896},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2978000044822693},{"id":"https://openalex.org/C2779328685","wikidata":"https://www.wikidata.org/wiki/Q1475557","display_name":"Patient safety","level":3,"score":0.29089999198913574},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.2906999886035919},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.2847000062465668},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.28349998593330383},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2815000116825104},{"id":"https://openalex.org/C2778149918","wikidata":"https://www.wikidata.org/wiki/Q3291156","display_name":"Population health","level":3,"score":0.2809000015258789},{"id":"https://openalex.org/C19648533","wikidata":"https://www.wikidata.org/wiki/Q691640","display_name":"Evidence-based medicine","level":3,"score":0.2687000036239624},{"id":"https://openalex.org/C63527458","wikidata":"https://www.wikidata.org/wiki/Q5133829","display_name":"Clinical decision support system","level":3,"score":0.266400009393692},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.262800008058548},{"id":"https://openalex.org/C3020452639","wikidata":"https://www.wikidata.org/wiki/Q454812","display_name":"Clinical judgment","level":2,"score":0.2596000134944916},{"id":"https://openalex.org/C2989236134","wikidata":"https://www.wikidata.org/wiki/Q31207","display_name":"Patient care","level":2,"score":0.25760000944137573},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.2556999921798706},{"id":"https://openalex.org/C202645933","wikidata":"https://www.wikidata.org/wiki/Q4930","display_name":"eHealth","level":3,"score":0.25060001015663147}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.25196","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.25196","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.25196","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.25196","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.7520906925201416}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Clinical":[0],"practice":[1],"guidelines":[2,176],"(CPGs)":[3],"play":[4],"a":[5,170,194,233],"pivotal":[6],"role":[7],"in":[8,24,62,78,166,257],"ensuring":[9],"evidence-based":[10],"decision-making":[11],"and":[12,36,57,73,119,153,177,236,252],"improving":[13],"patient":[14],"outcomes.":[15],"While":[16],"Large":[17],"Language":[18],"Models":[19],"(LLMs)":[20],"are":[21,240],"increasingly":[22],"deployed":[23],"healthcare":[25],"scenarios,":[26],"it":[27],"is":[28,109,208,247],"unclear":[29],"to":[30,38,115,164,180,219,223],"which":[31,214],"extend":[32],"LLMs":[33,61,217,256],"could":[34],"identify":[35],"adhere":[37,222],"CPGs":[39],"during":[40,224],"conversations.":[41,64,225],"To":[42,183,204],"address":[43],"this":[44],"gap,":[45],"we":[46,89,191],"introduce":[47],"CPGBench,":[48],"an":[49],"automated":[50],"framework":[51],"benchmarking":[52],"the":[53,79,117,129,146,150,175,185,209,250],"clinical":[54,92,215,229,238,260],"guideline":[55,151],"detection":[56,118],"adherence":[58,120,159],"capabilities":[59,121],"of":[60,122,187,255],"multi-turn":[63,107],"We":[65,126],"collect":[66],"3,418":[67],"CPG":[68],"documents":[69],"from":[70,162,201],"9":[71],"countries/regions":[72],"2":[74],"international":[75],"organizations":[76],"published":[77],"last":[80],"decade":[81],"spanning":[82],"across":[83],"24":[84],"specialties.":[85,203],"From":[86],"these":[87,245],"documents,":[88],"extract":[90],"32,155":[91],"recommendations":[93,131,216],"with":[94],"corresponding":[95,139],"publication":[96],"institute,":[97],"date,":[98],"country,":[99],"specialty,":[100],"recommendation":[101,113,230],"strength,":[102],"evidence":[103],"level,":[104],"etc.":[105],"One":[106],"conversation":[108],"generated":[110],"for":[111,249],"each":[112,228],"accordingly":[114],"evaluate":[116],"8":[123],"leading":[124],"LLMs.":[125],"find":[127],"that":[128,227,237],"71.1%-89.6%":[130],"can":[132,141],"be":[133,142],"correctly":[134,143],"detected,":[135],"while":[136],"only":[137],"3.6%-29.7%":[138],"titles":[140],"referenced,":[144],"revealing":[145,213],"gap":[147,172],"between":[148,173],"knowing":[149,174],"contents":[152],"where":[154],"they":[155],"come":[156],"from.":[157],"The":[158],"rates":[160],"range":[161],"21.8%":[163],"63.2%":[165],"different":[167,202],"models,":[168],"indicating":[169],"large":[171,234],"being":[178],"able":[179],"apply":[181],"them.":[182],"confirm":[184],"validity":[186],"our":[188,205],"automatic":[189],"analysis,":[190],"further":[192],"conduct":[193],"comprehensive":[195],"human":[196],"evaluation":[197],"involving":[198],"56":[199],"clinicians":[200],"knowledge,":[206],"CPGBench":[207],"first":[210],"benchmark":[211],"systematically":[212],"fail":[218],"detect":[220],"or":[221],"Given":[226],"may":[231],"affect":[232],"population":[235],"applications":[239],"inherently":[241],"safety":[242],"critical,":[243],"addressing":[244],"gaps":[246],"crucial":[248],"safe":[251],"responsible":[253],"deployment":[254],"real":[258],"world":[259],"practice.":[261]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-28T00:00:00"}