{"id":"https://openalex.org/W7223846594","doi":"https://doi.org/10.2196/preprints.115188","title":"Knowledge-Enhanced Large Language Models for Extracting Clinical Pathway Process Models From Guideline Text: Development and Evaluation Study (Preprint)","display_name":"Knowledge-Enhanced Large Language Models for Extracting Clinical Pathway Process Models From Guideline Text: Development and Evaluation Study (Preprint)","publication_year":2026,"publication_date":"2026-10-26","ids":{"openalex":"https://openalex.org/W7223846594","doi":"https://doi.org/10.2196/preprints.115188"},"language":"en","primary_location":{"id":"doi:10.2196/preprints.115188","doi":"https://doi.org/10.2196/preprints.115188","is_oa":false,"landing_page_url":"https://doi.org/10.2196/preprints.115188","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"volume":null,"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005125860","display_name":"William Van Woensel","orcid":"https://orcid.org/0000-0002-7049-8735","observed_orcids":["https://orcid.org/0000-0002-7049-8735"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"William Van Woensel","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-7049-8735","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135413630","display_name":"Alireza Houshidari","orcid":"https://orcid.org/0009-0006-0349-9574","observed_orcids":["https://orcid.org/0009-0006-0349-9574"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alireza Houshidari","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0009-0006-0349-9574","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043523779","display_name":"Daniel Amyot","orcid":"https://orcid.org/0000-0003-2414-1791","observed_orcids":["https://orcid.org/0000-0003-2414-1791"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Daniel Amyot","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-2414-1791","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017519118","display_name":"El Mostafa Bouattane","orcid":"https://orcid.org/0009-0008-4623-0887","observed_orcids":["https://orcid.org/0009-0008-4623-0887"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"El Mostafa Bouattane","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0009-0008-4623-0887","affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5118996656","display_name":"H\u00e9toum Misirliyan","orcid":"https://orcid.org/0009-0008-7213-184X","observed_orcids":["https://orcid.org/0009-0008-7213-184X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"H\u00e9toum Misirliyan","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0009-0008-7213-184X","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12664","display_name":"Clinical practice guidelines implementation","score":0.8065999746322632,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T12664","display_name":"Clinical practice guidelines implementation","score":0.8065999746322632,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10703","display_name":"Business Process Modeling and Analysis","score":0.06480000168085098,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.058400001376867294,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/care-pathways","display_name":"care pathways","score":0.8930000066757202},{"id":"https://openalex.org/keywords/process-model","display_name":"process model","score":0.7829999923706055},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.871999979019165},{"id":"https://openalex.org/keywords/knowledge-improvement","display_name":"knowledge improvement","score":0.6430000066757202},{"id":"https://openalex.org/keywords/prompt-injection","display_name":"prompt injection","score":0.6460000276565552},{"id":"https://openalex.org/keywords/formal-constraints","display_name":"formal constraints","score":0.5920000076293945},{"id":"https://openalex.org/keywords/linear-temporal-logic","display_name":"linear temporal logic","score":0.8799999952316284},{"id":"https://openalex.org/keywords/model-checking","display_name":"model checking","score":0.8009999990463257},{"id":"https://openalex.org/keywords/clinical-decision-support","display_name":"clinical decision support","score":0.7649999856948853},{"id":"https://openalex.org/keywords/community-acquired-pneumonia","display_name":"community-acquired pneumonia","score":0.6830000281333923},{"id":"https://openalex.org/keywords/structural-similarity","display_name":"structural similarity","score":0.4830000102519989},{"id":"https://openalex.org/keywords/ontological-relations","display_name":"ontological relations","score":0.6639999747276306},{"id":"https://openalex.org/keywords/task-dependencies","display_name":"task dependencies","score":0.421999990940094},{"id":"https://openalex.org/keywords/patient-safety","display_name":"patient safety","score":0.4880000054836273},{"id":"https://openalex.org/keywords/gpt-5-6","display_name":"GPT-5.6","score":0.6150000095367432}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7286999821662903},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.625},{"id":"https://openalex.org/C2775948798","wikidata":"https://www.wikidata.org/wiki/Q5160261","display_name":"Conformance checking","level":5,"score":0.5996999740600586},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5548999905586243},{"id":"https://openalex.org/C2778373050","wikidata":"https://www.wikidata.org/wiki/Q1774706","display_name":"Clinical pathway","level":2,"score":0.4731000065803528},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.46799999475479126},{"id":"https://openalex.org/C179603123","wikidata":"https://www.wikidata.org/wiki/Q1941921","display_name":"Modeling language","level":3,"score":0.4645000100135803},{"id":"https://openalex.org/C76956256","wikidata":"https://www.wikidata.org/wiki/Q27610560","display_name":"Process modeling","level":3,"score":0.45170000195503235},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44769999384880066},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4156999886035919},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.41429999470710754},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.3817000091075897},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.38029998540878296},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3522000014781952},{"id":"https://openalex.org/C124670913","wikidata":"https://www.wikidata.org/wiki/Q2608526","display_name":"Process mining","level":5,"score":0.34220001101493835},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.3409000039100647},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3384999930858612},{"id":"https://openalex.org/C2777220311","wikidata":"https://www.wikidata.org/wiki/Q6423340","display_name":"Knowledge acquisition","level":2,"score":0.337799996137619},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3328999876976013},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.32690000534057617},{"id":"https://openalex.org/C2780182762","wikidata":"https://www.wikidata.org/wiki/Q1630279","display_name":"Guideline","level":2,"score":0.3181000053882599},{"id":"https://openalex.org/C2775966667","wikidata":"https://www.wikidata.org/wiki/Q6423384","display_name":"Knowledge modeling","level":3,"score":0.3043000102043152},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.3009999990463257},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.2890999913215637},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.2824999988079071},{"id":"https://openalex.org/C145644426","wikidata":"https://www.wikidata.org/wiki/Q169411","display_name":"Unified Modeling Language","level":3,"score":0.2773999869823456},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.2752000093460083},{"id":"https://openalex.org/C120567893","wikidata":"https://www.wikidata.org/wiki/Q1582085","display_name":"Knowledge extraction","level":2,"score":0.27300000190734863},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.27129998803138733},{"id":"https://openalex.org/C2775907273","wikidata":"https://www.wikidata.org/wiki/Q7805281","display_name":"Time constraint","level":2,"score":0.2669000029563904}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.2196/preprints.115188","doi":"https://doi.org/10.2196/preprints.115188","is_oa":false,"landing_page_url":"https://doi.org/10.2196/preprints.115188","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Good health and well-being","score":0.5683000087738037,"id":"https://metadata.un.org/sdg/3"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":44,"referenced_works":["https://openalex.org/W978581747","https://openalex.org/W2023808162","https://openalex.org/W2042759024","https://openalex.org/W2045766845","https://openalex.org/W2101107197","https://openalex.org/W2103209067","https://openalex.org/W2156233842","https://openalex.org/W2162864452","https://openalex.org/W2260214316","https://openalex.org/W2293099500","https://openalex.org/W2800575625","https://openalex.org/W2884618163","https://openalex.org/W2903265272","https://openalex.org/W3030845798","https://openalex.org/W3041597941","https://openalex.org/W3096943904","https://openalex.org/W3174346744","https://openalex.org/W4245505136","https://openalex.org/W4281702050","https://openalex.org/W4285271077","https://openalex.org/W4285303785","https://openalex.org/W4312911974","https://openalex.org/W4367185264","https://openalex.org/W4376879040","https://openalex.org/W4390694561","https://openalex.org/W4391407054","https://openalex.org/W4391640544","https://openalex.org/W4392781032","https://openalex.org/W4401415471","https://openalex.org/W4402397383","https://openalex.org/W4403689435","https://openalex.org/W4403793823","https://openalex.org/W4407554595","https://openalex.org/W4409779994","https://openalex.org/W4413978037","https://openalex.org/W4414322093","https://openalex.org/W4414359769","https://openalex.org/W4414977039","https://openalex.org/W4416092103","https://openalex.org/W7122774503","https://openalex.org/W7126202456","https://openalex.org/W7160232981","https://openalex.org/W7161170198","https://openalex.org/W7164859548"],"related_works":["https://openalex.org/W7108318608","https://openalex.org/W7161170198","https://openalex.org/W4407554595","https://openalex.org/W7160232981","https://openalex.org/W4414340659","https://openalex.org/W7155210959","https://openalex.org/W4405944060","https://openalex.org/W7126202456","https://openalex.org/W7161958316","https://openalex.org/W7161562862"],"abstract_inverted_index":{"BACKGROUND":[0],"Clinical":[1],"pathways":[2,205],"(CPW)":[3],"turn":[4],"evidence-based":[5],"guidelines":[6,282,305],"into":[7,121,175,352],"step-by-step":[8],"care":[9,27],"protocols.":[10],"When":[11],"formalized":[12,115,126,349],"as":[13,70,116,129,139,400],"process":[14,35,103,179,200],"models,":[15],"they":[16],"can":[17,33,78],"support":[18,22],"automated":[19],"clinical":[20,106,204,365],"decision":[21],"and":[23,64,74,81,95,119,138,153,156,159,173,250,283,302,306,368,388,410,432],"conformance":[24,322,390],"checking":[25],"of":[26,47,86,183,336,363],"trajectories.":[28],"Large":[29],"language":[30,172],"models":[31,36,104,201,244,367],"(LLMs)":[32],"extract":[34],"from":[37,56,105,202,275,421],"text":[38,409],"at":[39,318],"scale,":[40],"but":[41],"their":[42,389],"output":[43,185],"still":[44],"falls":[45],"short":[46],"clinician-validated":[48],"reference":[49,243],"models.":[50],"This":[51,90],"gap":[52],"stems":[53],"in":[54,84,131,170,206,234,339,392],"part":[55],"missing":[57,111],"context:":[58],"CPW":[59],"are":[60,383],"written":[61],"for":[62,101,231,334,402],"experts":[63],"often":[65],"omit":[66],"background":[67,112,127,350],"knowledge":[68,98,113,128,164,223,286,308,327,351],"such":[69],"task":[71,145,373],"conflicts,":[72,146],"dependencies,":[73,148],"priorities.":[75],"These":[76],"omissions":[77],"compromise":[79],"fidelity":[80,362],"patient":[82],"safety":[83],"case":[85],"LLM-based":[87],"extraction.":[88],"OBJECTIVE":[89],"study":[91,213],"aimed":[92],"to":[93,242,278,324,359,369,385,429],"develop":[94],"evaluate":[96],"a":[97,163,261,297,355,378,403],"enhancement":[99,165],"approach":[100],"extracting":[102],"pathways.":[107],"In":[108,162],"this":[109],"approach,":[110,166],"is":[114,354],"declarative":[117],"constraints":[118,130,143,167,188,287,309,331,345,376,382,412,419],"injected":[120,174],"LLM":[122],"prompts.":[123],"METHODS":[124],"We":[125],"Linear":[132],"Temporal":[133],"Logic":[134],"over":[135],"finite":[136],"traces":[137],"ontological":[140],"relations.":[141],"The":[142],"covered":[144],"decisional":[147],"prioritized":[149],"conditional":[150],"tasks,":[151,155],"alternative":[152],"part-of":[154],"pathway":[157,316,366],"entry":[158],"exit":[160],"points.":[161],"were":[168,237,255,332],"narrated":[169],"natural":[171],"the":[176,184,187,304,307,340,361,386,393,430],"prompt":[177,216],"alongside":[178],"modelling":[180,220,281],"guidelines.":[181],"Conformance":[182],"with":[186,264,280,285],"was":[189],"then":[190],"verified":[191],"by":[192,239,246,251],"model":[193],"checking.":[194],"Using":[195],"GPT":[196],"5.6,":[197],"we":[198],"extracted":[199],"8":[203,337],"Ontario\u2019s":[207],"Quality-Based":[208],"Procedures":[209],"repository.":[210],"An":[211],"ablation":[212],"compared":[214,256],"3":[215],"conditions":[217],"(baseline,":[218],"+":[219,222],"guidelines,":[221],"constraints),":[224],"each":[225,319],"run":[226],"5":[227],"times":[228],"per":[229],"pathway,":[230,343],"120":[232],"extractions":[233],"total.":[235],"Outputs":[236],"evaluated":[238],"structural":[240,272],"similarity":[241,273],"validated":[245],"2":[247],"emergency":[248],"physicians,":[249],"constraint":[252],"conformance.":[253],"Conditions":[254],"on":[257,427],"per-pathway":[258],"means":[259],"using":[260],"Friedman":[262],"test":[263],"exact,":[265],"Holm-corrected":[266],"Wilcoxon":[267],"signed-rank":[268],"tests.":[269],"RESULTS":[270],"Mean":[271],"rose":[274],"0.60":[276],"(baseline)":[277],"0.76":[279],"0.84":[284],"(absolute":[288],"gain,":[289],"0.24;":[290],"40.2%":[291],"relative":[292],"increase).":[293],"Prompt":[294],"condition":[295],"had":[296],"significant":[298,311],"effect":[299],"(\u03c7_2^2=16.0,":[300],"P<.001),":[301],"both":[303],"gave":[310],"stepwise":[312],"improvements":[313],"(P=.02).":[314],"Every":[315],"improved":[317,323],"step.":[320],"Constraint":[321],"0.98":[325],"after":[326],"injection":[328],"(P=.008).":[329],"All":[330],"met":[333],"7":[335],"pathways;":[338],"community-acquired":[341],"pneumonia":[342],"two":[344],"conflicted.":[346],"CONCLUSIONS":[347],"Injecting":[348],"prompts":[353],"practical,":[356],"model-agnostic":[357],"way":[358],"improve":[360],"LLM-extracted":[364],"guard":[370],"against":[371],"unsafe":[372],"sequences.":[374],"Formal":[375],"allow":[377],"feedback":[379],"loop,":[380],"where":[381],"applied":[384],"input":[387,401],"checked":[391],"output;":[394],"non-conformance":[395],"could":[396],"be":[397],"used":[398],"further":[399],"next":[404],"iteration.":[405],"Highly":[406],"complex":[407],"guideline":[408],"conflicting":[411],"remain":[413],"challenges.":[414],"Future":[415],"work":[416],"will":[417],"distill":[418],"automatically":[420],"public":[422],"sources,":[423],"inject":[424],"them":[425],"based":[426],"relevance":[428],"CPW,":[431],"involve":[433],"large-scale":[434],"evaluations.":[435]},"counts_by_year":[],"updated_date":"2026-10-29T09:56:32.521200","created_date":"2026-10-28T00:00:00"}