{"id":"https://openalex.org/W4401754091","doi":"https://doi.org/10.1109/ichi61247.2024.00108","title":"Multimodal Clinical Prediction with Unified Prompts and Pretrained Large-Language Models","display_name":"Multimodal Clinical Prediction with Unified Prompts and Pretrained Large-Language Models","publication_year":2024,"publication_date":"2024-06-03","ids":{"openalex":"https://openalex.org/W4401754091","doi":"https://doi.org/10.1109/ichi61247.2024.00108"},"language":"en","primary_location":{"id":"doi:10.1109/ichi61247.2024.00108","doi":"https://doi.org/10.1109/ichi61247.2024.00108","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ichi61247.2024.00108","pdf_url":null,"source":{"id":"https://openalex.org/S4363608310","display_name":"IEEE International Conference on Healthcare Informatics (ICHI)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE 12th International Conference on Healthcare Informatics (ICHI)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047763022","display_name":"Caleb Winston","orcid":"https://orcid.org/0000-0001-8458-9438","observed_orcids":["https://orcid.org/0000-0001-8458-9438"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Caleb Winston","raw_affiliation_strings":["Perelman School of Medicine, University of Pennsylvania,Philadelphia,USA","Stanford University,Department of Computer Science,Stanford,USA","University of Washington,Department of Computer Science,Seattle,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Perelman School of Medicine, University of Pennsylvania,Philadelphia,USA","institution_ids":["https://openalex.org/I79576946"]},{"raw_affiliation_string":"Stanford University,Department of Computer Science,Stanford,USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"University of Washington,Department of Computer Science,Seattle,USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080955146","display_name":"Chloe N. Winston","orcid":"https://orcid.org/0000-0002-6856-2945","observed_orcids":["https://orcid.org/0000-0002-6856-2945"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chloe Winston","raw_affiliation_strings":["Perelman School of Medicine, University of Pennsylvania,Philadelphia,USA","Stanford University,Department of Computer Science,Stanford,USA","University of Washington,Department of Computer Science,Seattle,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Perelman School of Medicine, University of Pennsylvania,Philadelphia,USA","institution_ids":["https://openalex.org/I79576946"]},{"raw_affiliation_string":"Stanford University,Department of Computer Science,Stanford,USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"University of Washington,Department of Computer Science,Seattle,USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063151961","display_name":"Cailin Winston","orcid":"https://orcid.org/0000-0002-3766-1254","observed_orcids":["https://orcid.org/0000-0002-3766-1254"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cailin Winston","raw_affiliation_strings":["Perelman School of Medicine, University of Pennsylvania,Philadelphia,USA","Stanford University,Department of Computer Science,Stanford,USA","University of Washington,Department of Computer Science,Seattle,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Perelman School of Medicine, University of Pennsylvania,Philadelphia,USA","institution_ids":["https://openalex.org/I79576946"]},{"raw_affiliation_string":"Stanford University,Department of Computer Science,Stanford,USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"University of Washington,Department of Computer Science,Seattle,USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033475814","display_name":"Claris Winston","orcid":"https://orcid.org/0000-0002-1342-5257","observed_orcids":["https://orcid.org/0000-0002-1342-5257"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Claris Winston","raw_affiliation_strings":["Perelman School of Medicine, University of Pennsylvania,Philadelphia,USA","Stanford University,Department of Computer Science,Stanford,USA","University of Washington,Department of Computer Science,Seattle,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Perelman School of Medicine, University of Pennsylvania,Philadelphia,USA","institution_ids":["https://openalex.org/I79576946"]},{"raw_affiliation_string":"Stanford University,Department of Computer Science,Stanford,USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"University of Washington,Department of Computer Science,Seattle,USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025400969","display_name":"Cleah Winston","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cleah Winston","raw_affiliation_strings":["Perelman School of Medicine, University of Pennsylvania,Philadelphia,USA","Stanford University,Department of Computer Science,Stanford,USA","University of Washington,Department of Computer Science,Seattle,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Perelman School of Medicine, University of Pennsylvania,Philadelphia,USA","institution_ids":["https://openalex.org/I79576946"]},{"raw_affiliation_string":"Stanford University,Department of Computer Science,Stanford,USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"University of Washington,Department of Computer Science,Seattle,USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.105,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.82032892,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"679","last_page":"683"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11396","display_name":"Artificial Intelligence in Healthcare","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/3605","display_name":"Health Information Management"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9150000214576721},{"id":"https://openalex.org/keywords/chief-complaint","display_name":"chief complaint","score":0.6470000147819519},{"id":"https://openalex.org/keywords/emergency-department-visits","display_name":"emergency department visits","score":0.7049999833106995},{"id":"https://openalex.org/keywords/clinical-outcome-prediction","display_name":"clinical outcome prediction","score":0.6779999732971191},{"id":"https://openalex.org/keywords/hospital-admission-prediction","display_name":"hospital admission prediction","score":0.8140000104904175},{"id":"https://openalex.org/keywords/length-of-hospital-stay","display_name":"length of hospital stay","score":0.828000009059906},{"id":"https://openalex.org/keywords/multimodal-fusion","display_name":"multimodal fusion","score":0.5479999780654907}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7659487724304199},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.49673253297805786},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4644384980201721},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.46418967843055725}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ichi61247.2024.00108","doi":"https://doi.org/10.1109/ichi61247.2024.00108","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ichi61247.2024.00108","pdf_url":null,"source":{"id":"https://openalex.org/S4363608310","display_name":"IEEE International Conference on Healthcare Informatics (ICHI)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE 12th International Conference on Healthcare Informatics (ICHI)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/3","display_name":"Good health and well-being","score":0.9391000270843506}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W2002108805","https://openalex.org/W2034741975","https://openalex.org/W2052808156","https://openalex.org/W2105004565","https://openalex.org/W2137073380","https://openalex.org/W2588147029","https://openalex.org/W2808524443","https://openalex.org/W2896457183","https://openalex.org/W2937845937","https://openalex.org/W2963341956","https://openalex.org/W2963691377","https://openalex.org/W2965163738","https://openalex.org/W2989268584","https://openalex.org/W3005680577","https://openalex.org/W3032269864","https://openalex.org/W3046375318","https://openalex.org/W3095791468","https://openalex.org/W4210494085","https://openalex.org/W4225659209","https://openalex.org/W4293258565","https://openalex.org/W4313439128","https://openalex.org/W4380319195","https://openalex.org/W4386741020","https://openalex.org/W4388860805"],"related_works":["https://openalex.org/W7169923492","https://openalex.org/W7213939893","https://openalex.org/W4417156540","https://openalex.org/W7223172290","https://openalex.org/W7208815985","https://openalex.org/W7166651509","https://openalex.org/W4388037391","https://openalex.org/W4415956540","https://openalex.org/W4362655286","https://openalex.org/W4405841623"],"abstract_inverted_index":{"Clinical":[0],"prediction":[1,113,188],"models":[2,91,163],"(CPMs)":[3],"increasingly":[4],"rely":[5],"on":[6,111],"multiple":[7,84],"modalities":[8],"to":[9,40,45,59,98,200],"predict":[10],"clinical":[11,105,190],"outcomes.":[12],"The":[13],"use":[14,88,150,166],"of":[15,30,74,114,117,189,196],"free-text":[16,125],"data":[17,169],"sources":[18],"(e.g.,":[19],"chief":[20,126,152],"complaint,":[21],"medical":[22],"notes)":[23],"presents":[24],"new":[25,72],"challenges":[26],"in":[27,179,186],"the":[28,31,38,42,100,112,151,167,187],"heterogeneity":[29],"text":[32,44,78],"across":[33],"providers":[34],"and":[35,37,103,128,135,162,198],"patients":[36],"need":[39],"convert":[41],"free":[43],"numerical":[46,130,172],"features":[47],"before":[48],"combining":[49],"with":[50,94,143],"other":[51],"modalities.":[52,65,85],"Prior":[53],"work":[54],"has":[55],"employed":[56],"multi-head":[57],"architectures":[58],"learn":[60],"separate":[61],"heads":[62],"for":[63],"different":[64],"In":[66],"this":[67],"work,":[68],"we":[69],"propose":[70],"a":[71,76,115,124],"approach":[73,110],"constructing":[75],"unified":[77,101,144],"prompt":[79,102],"that":[80,140,148,164],"captures":[81],"information":[82,134],"from":[83,119],"We":[86,107,138,182],"then":[87],"existing":[89],"large-language":[90],"(LLMs),":[92],"optimized":[93,141],"diagnosis-contrastive":[95],"learning":[96],"(DCL)":[97],"encode":[99],"make":[104],"predictions.":[106],"test":[108],"our":[109],"variety":[116],"outcomes":[118],"emergency":[120],"department":[121],"visits":[122],"using":[123],"complaint":[127,153],"structured":[129,168],"data,":[131],"including":[132,192],"demographic":[133],"vital":[136],"signs.":[137],"find":[139],"LLMs":[142,147],"prompts":[145],"outperform":[146],"only":[149,165],"by":[154,174],"0.02":[155],"weighted":[156],"F1":[157],"score":[158],"(p":[159,176],"<":[160,177],"0.0001)":[161,178],"modality":[170],"as":[171],"inputs":[173],"0.15":[175],"predicting":[180],"acuity.":[181],"also":[183],"observe":[184],"improvements":[185],"outcomes,":[191],"hospital":[193],"admission,":[194],"length":[195],"stay,":[197],"time":[199],"revisit.":[201]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":2}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2024-08-23T00:00:00"}