{"id":"https://openalex.org/W4403598801","doi":"https://doi.org/10.1109/tmi.2024.3484294","title":"Disease-Informed Adaptation of Vision-Language Models","display_name":"Disease-Informed Adaptation of Vision-Language Models","publication_year":2024,"publication_date":"2024-10-21","ids":{"openalex":"https://openalex.org/W4403598801","doi":"https://doi.org/10.1109/tmi.2024.3484294","pmid":"https://pubmed.ncbi.nlm.nih.gov/39437273"},"language":"en","primary_location":{"id":"doi:10.1109/tmi.2024.3484294","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmi.2024.3484294","pdf_url":null,"source":{"id":"https://openalex.org/S58069681","display_name":"IEEE Transactions on Medical Imaging","issn_l":"0278-0062","issn":["0278-0062","1558-254X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Medical Imaging","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030694927","display_name":"Jiajin Zhang","orcid":"https://orcid.org/0000-0001-6516-4012"},"institutions":[{"id":"https://openalex.org/I165799507","display_name":"Rensselaer Polytechnic Institute","ror":"https://ror.org/01rtyzb94","country_code":"US","type":"education","lineage":["https://openalex.org/I165799507"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jiajin Zhang","raw_affiliation_strings":["Department of Biomedical Engineering and the Center for Biotechnology and Interdisciplinary Studies, Rensselaer Polytechnic Institute, Troy, NY, USA","Department of Biomedical Engineering and Center for Biotechnology and Interdisciplinary Studies, Rensselaer Polytechnic Institute, Troy, NY, USA"],"affiliations":[{"raw_affiliation_string":"Department of Biomedical Engineering and the Center for Biotechnology and Interdisciplinary Studies, Rensselaer Polytechnic Institute, Troy, NY, USA","institution_ids":["https://openalex.org/I165799507"]},{"raw_affiliation_string":"Department of Biomedical Engineering and Center for Biotechnology and Interdisciplinary Studies, Rensselaer Polytechnic Institute, Troy, NY, USA","institution_ids":["https://openalex.org/I165799507"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100400458","display_name":"Ge Wang","orcid":"https://orcid.org/0000-0002-2656-7705"},"institutions":[{"id":"https://openalex.org/I165799507","display_name":"Rensselaer Polytechnic Institute","ror":"https://ror.org/01rtyzb94","country_code":"US","type":"education","lineage":["https://openalex.org/I165799507"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ge Wang","raw_affiliation_strings":["Department of Biomedical Engineering and the Center for Biotechnology and Interdisciplinary Studies, Rensselaer Polytechnic Institute, Troy, NY, USA","Department of Biomedical Engineering and Center for Biotechnology and Interdisciplinary Studies, Rensselaer Polytechnic Institute, Troy, NY, USA"],"affiliations":[{"raw_affiliation_string":"Department of Biomedical Engineering and the Center for Biotechnology and Interdisciplinary Studies, Rensselaer Polytechnic Institute, Troy, NY, USA","institution_ids":["https://openalex.org/I165799507"]},{"raw_affiliation_string":"Department of Biomedical Engineering and Center for Biotechnology and Interdisciplinary Studies, Rensselaer Polytechnic Institute, Troy, NY, USA","institution_ids":["https://openalex.org/I165799507"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087606400","display_name":"Mannudeep K. Kalra","orcid":"https://orcid.org/0000-0001-9938-7476"},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]},{"id":"https://openalex.org/I4210087915","display_name":"Massachusetts General Hospital","ror":"https://ror.org/002pd6e78","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210087915","https://openalex.org/I48633490"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mannudeep K. Kalra","raw_affiliation_strings":["Department of Radiology, Massachusetts General Hospital, Harvard Medical School, Boston, MA, USA"],"affiliations":[{"raw_affiliation_string":"Department of Radiology, Massachusetts General Hospital, Harvard Medical School, Boston, MA, USA","institution_ids":["https://openalex.org/I4210087915","https://openalex.org/I136199984"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5054075262","display_name":"Pingkun Yan","orcid":"https://orcid.org/0000-0002-9779-2141"},"institutions":[{"id":"https://openalex.org/I165799507","display_name":"Rensselaer Polytechnic Institute","ror":"https://ror.org/01rtyzb94","country_code":"US","type":"education","lineage":["https://openalex.org/I165799507"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pingkun Yan","raw_affiliation_strings":["Department of Biomedical Engineering and the Center for Biotechnology and Interdisciplinary Studies, Rensselaer Polytechnic Institute, Troy, NY, USA","Department of Biomedical Engineering and Center for Biotechnology and Interdisciplinary Studies, Rensselaer Polytechnic Institute, Troy, NY, USA"],"affiliations":[{"raw_affiliation_string":"Department of Biomedical Engineering and the Center for Biotechnology and Interdisciplinary Studies, Rensselaer Polytechnic Institute, Troy, NY, USA","institution_ids":["https://openalex.org/I165799507"]},{"raw_affiliation_string":"Department of Biomedical Engineering and Center for Biotechnology and Interdisciplinary Studies, Rensselaer Polytechnic Institute, Troy, NY, USA","institution_ids":["https://openalex.org/I165799507"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5030694927"],"corresponding_institution_ids":["https://openalex.org/I165799507"],"apc_list":null,"apc_paid":null,"fwci":3.3281,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.92929192,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":"44","issue":"10","first_page":"3984","last_page":"3996"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14235","display_name":"Linguistic, Cultural, and Literary Studies","score":0.4830999970436096,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T14235","display_name":"Linguistic, Cultural, and Literary Studies","score":0.4830999970436096,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6764965057373047},{"id":"https://openalex.org/keywords/adaptation","display_name":"Adaptation (eye)","score":0.6598200798034668},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4917352497577667},{"id":"https://openalex.org/keywords/disease","display_name":"Disease","score":0.48111432790756226},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.4773419201374054},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3911626935005188},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.18801423907279968},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.18453863263130188},{"id":"https://openalex.org/keywords/neuroscience","display_name":"Neuroscience","score":0.11651098728179932},{"id":"https://openalex.org/keywords/pathology","display_name":"Pathology","score":0.08679580688476562}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6764965057373047},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.6598200798034668},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4917352497577667},{"id":"https://openalex.org/C2779134260","wikidata":"https://www.wikidata.org/wiki/Q12136","display_name":"Disease","level":2,"score":0.48111432790756226},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4773419201374054},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3911626935005188},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.18801423907279968},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.18453863263130188},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.11651098728179932},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.08679580688476562}],"mesh":[{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D007090","descriptor_name":"Image Interpretation, Computer-Assisted","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D007090","descriptor_name":"Image Interpretation, Computer-Assisted","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true}],"locations_count":2,"locations":[{"id":"doi:10.1109/tmi.2024.3484294","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmi.2024.3484294","pdf_url":null,"source":{"id":"https://openalex.org/S58069681","display_name":"IEEE Transactions on Medical Imaging","issn_l":"0278-0062","issn":["0278-0062","1558-254X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Medical Imaging","raw_type":"journal-article"},{"id":"pmid:39437273","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/39437273","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on medical imaging","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6946403354","display_name":null,"funder_award_id":"2046708","funder_id":"https://openalex.org/F4320337377","funder_display_name":"Office of Advanced Cyberinfrastructure"},{"id":"https://openalex.org/G8931101168","display_name":null,"funder_award_id":"CAREER Award OAC 2046708","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320337377","display_name":"Office of Advanced Cyberinfrastructure","ror":"https://ror.org/04nh1dc89"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":39,"referenced_works":["https://openalex.org/W2108598243","https://openalex.org/W2165698076","https://openalex.org/W2887280559","https://openalex.org/W2887782657","https://openalex.org/W2914203365","https://openalex.org/W2963026768","https://openalex.org/W2963466845","https://openalex.org/W2971296908","https://openalex.org/W2995225687","https://openalex.org/W3013277995","https://openalex.org/W3097760447","https://openalex.org/W3105081694","https://openalex.org/W3105976960","https://openalex.org/W3134329288","https://openalex.org/W3135057764","https://openalex.org/W3139487216","https://openalex.org/W3170581063","https://openalex.org/W3198377975","https://openalex.org/W3201906559","https://openalex.org/W3211326549","https://openalex.org/W4213442487","https://openalex.org/W4223899585","https://openalex.org/W4288060296","https://openalex.org/W4296027312","https://openalex.org/W4312310776","https://openalex.org/W4312533035","https://openalex.org/W4312956471","https://openalex.org/W4321021041","https://openalex.org/W4385573131","https://openalex.org/W4385948838","https://openalex.org/W4386065763","https://openalex.org/W4386071547","https://openalex.org/W4386790226","https://openalex.org/W4387211400","https://openalex.org/W4387225969","https://openalex.org/W4387968176","https://openalex.org/W4392947532","https://openalex.org/W4403069340","https://openalex.org/W4406983345"],"related_works":["https://openalex.org/W2772917594","https://openalex.org/W2036807459","https://openalex.org/W2058170566","https://openalex.org/W2755342338","https://openalex.org/W2166024367","https://openalex.org/W3116076068","https://openalex.org/W2229312674","https://openalex.org/W2951359407","https://openalex.org/W2079911747","https://openalex.org/W1969923398"],"abstract_inverted_index":{"Expertise":[0],"scarcity":[1],"and":[2,168],"high":[3],"cost":[4],"of":[5,11,61,75,90,112,120,131,153],"data":[6],"annotation":[7],"hinder":[8],"the":[9,28,34,58,72,76,117,127,151,154,172],"development":[10],"artificial":[12],"intelligence":[13],"(AI)":[14],"foundation":[15,30,91],"models":[16,31,39],"for":[17,64,87],"medical":[18,166],"image":[19],"analysis.":[20],"Transfer":[21],"learning":[22,79,89,116,143],"provides":[23],"a":[24,65,84,100,105,139],"way":[25],"to":[26,32,44,71,95,99,148,178],"utilize":[27],"off-the-shelf":[29],"address":[33],"clinical":[35],"challenges.":[36],"However,":[37],"such":[38],"encounter":[40],"difficulties":[41],"when":[42],"adapting":[43],"new":[45,66,101,121,155],"diseases":[46],"not":[47],"presented":[48],"in":[49,138,175],"their":[50],"original":[51],"pre-training":[52],"datasets.":[53],"Compounding":[54],"this":[55],"challenge":[56],"is":[57],"limited":[59,159],"availability":[60],"example":[62],"cases":[63],"disease,":[67,156],"which":[68,145],"further":[69],"leads":[70],"poor":[73],"performance":[74,176],"existing":[77,180],"transfer":[78,88],"techniques.":[80,182],"This":[81],"paper":[82],"proposes":[83],"novel":[85,140],"method":[86],"Vision-Language":[92],"Models":[93],"(VLMs)":[94],"efficiently":[96],"adapt":[97],"them":[98],"disease":[102,122,141],"with":[103,158],"only":[104],"few":[106],"examples.":[107],"Such":[108],"an":[109],"effective":[110],"adaptation":[111,181],"VLMs":[113,147,167],"hinges":[114],"on":[115,126],"nuanced":[118],"representation":[119],"concepts.":[123],"By":[124],"capitalizing":[125],"joint":[128],"visual-linguistic":[129],"capabilities":[130],"VLMs,":[132],"we":[133],"introduce":[134],"disease-informed":[135],"contextual":[136],"prompting":[137],"prototype":[142],"framework,":[144],"enables":[146],"quickly":[149],"grasp":[150],"concept":[152],"even":[157],"data.":[160],"Extensive":[161],"experiments":[162],"across":[163],"multiple":[164,169],"pre-trained":[165],"tasks":[170],"showcase":[171],"notable":[173],"enhancements":[174],"compared":[177],"other":[179],"The":[183],"code":[184],"will":[185],"be":[186],"made":[187],"publicly":[188],"available":[189],"at":[190],"https://github.com/RPIDIAL/Disease-informed-VLM-Adaptation.":[191]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-10-10T00:00:00"}