{"id":"https://openalex.org/W4406259912","doi":"https://doi.org/10.1109/bibm62325.2024.10822357","title":"Aligning Multimodal Biomedical Images and Language via One Large Vision-Language Model","display_name":"Aligning Multimodal Biomedical Images and Language via One Large Vision-Language Model","publication_year":2024,"publication_date":"2024-12-03","ids":{"openalex":"https://openalex.org/W4406259912","doi":"https://doi.org/10.1109/bibm62325.2024.10822357"},"language":"en","primary_location":{"id":"doi:10.1109/bibm62325.2024.10822357","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bibm62325.2024.10822357","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Bioinformatics and Biomedicine (BIBM)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100707830","display_name":"Haojie Zhang","orcid":"https://orcid.org/0000-0002-4577-3600"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haojie Zhang","raw_affiliation_strings":["Central South University,School of Computer Science and Engineering,Changsha,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University,School of Computer Science and Engineering,Changsha,China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033885404","display_name":"Min Zeng","orcid":"https://orcid.org/0000-0002-1726-0955"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Min Zeng","raw_affiliation_strings":["Central South University,School of Computer Science and Engineering,Changsha,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University,School of Computer Science and Engineering,Changsha,China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054556534","display_name":"Jinfeng Ding","orcid":null},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jinfeng Ding","raw_affiliation_strings":["Central South University,Xiangya School of Nursing,Changsha,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University,Xiangya School of Nursing,Changsha,China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004436217","display_name":"Yixiong Liang","orcid":"https://orcid.org/0000-0003-0407-5838"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yixiong Liang","raw_affiliation_strings":["Central South University,School of Computer Science and Engineering,Changsha,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University,School of Computer Science and Engineering,Changsha,China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044328870","display_name":"Ruiqing Zheng","orcid":"https://orcid.org/0000-0001-6372-6798"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruiqing Zheng","raw_affiliation_strings":["Central South University,School of Computer Science and Engineering,Changsha,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University,School of Computer Science and Engineering,Changsha,China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055017398","display_name":"Zhe Qu","orcid":"https://orcid.org/0000-0001-8568-0460"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhe Qu","raw_affiliation_strings":["Central South University,School of Computer Science and Engineering,Changsha,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University,School of Computer Science and Engineering,Changsha,China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100719013","display_name":"Min Li","orcid":"https://orcid.org/0000-0002-0188-1394"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Min Li","raw_affiliation_strings":["Central South University,School of Computer Science and Engineering,Changsha,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University,School of Computer Science and Engineering,Changsha,China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025341442","display_name":"Shichao Kan","orcid":"https://orcid.org/0000-0003-0097-6196"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shichao Kan","raw_affiliation_strings":["Central South University,School of Computer Science and Engineering,Changsha,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University,School of Computer Science and Engineering,Changsha,China","institution_ids":["https://openalex.org/I139660479"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I139660479"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2874","last_page":"2879"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.6929000020027161,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.6929000020027161,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7580914497375488},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5480100512504578},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4765492081642151},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4389653205871582},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.32107383012771606}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7580914497375488},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5480100512504578},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4765492081642151},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4389653205871582},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.32107383012771606}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bibm62325.2024.10822357","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bibm62325.2024.10822357","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Bioinformatics and Biomedicine (BIBM)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321514","display_name":"Central South University","ror":"https://ror.org/00f1zfq44"},{"id":"https://openalex.org/F4320322843","display_name":"Natural Science Foundation of\u00a0Hunan Province","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W2901466771","https://openalex.org/W3165058054","https://openalex.org/W3175183715","https://openalex.org/W4382517509","https://openalex.org/W4399567248","https://openalex.org/W4402727574","https://openalex.org/W4402727922","https://openalex.org/W4403730271","https://openalex.org/W4404783556","https://openalex.org/W6784910178","https://openalex.org/W6791353385","https://openalex.org/W6796581206","https://openalex.org/W6851592950","https://openalex.org/W6852032383","https://openalex.org/W6852629218","https://openalex.org/W6852751528","https://openalex.org/W6853469104","https://openalex.org/W6856794988"],"related_works":["https://openalex.org/W2772917594","https://openalex.org/W2036807459","https://openalex.org/W2058170566","https://openalex.org/W2755342338","https://openalex.org/W2166024367","https://openalex.org/W3116076068","https://openalex.org/W2229312674","https://openalex.org/W2951359407","https://openalex.org/W2079911747","https://openalex.org/W1969923398"],"abstract_inverted_index":{"Large":[0],"Vision-Language":[1],"Models":[2],"(LVLMs)":[3],"have":[4],"garnered":[5],"substantial":[6],"attention":[7],"in":[8,178],"the":[9,81,88,115,133,165,172],"biomedical":[10,40,52,96,123,181],"image":[11,73,124],"analysis":[12,125,135],"domain":[13],"due":[14],"to":[15],"their":[16],"robust":[17],"vision":[18,82],"understanding":[19],"capabilities.":[20],"However,":[21],"current":[22],"methods":[23],"rely":[24],"heavily":[25],"on":[26,114,121,147,155],"dataset-":[27],"and":[28,39,54,76,84,118,130,139,153],"modality-specific":[29],"fine-tuning.":[30],"This":[31],"involves":[32],"tuning":[33],"separate":[34],"models":[35],"for":[36,49],"each":[37],"dataset":[38,117],"modality.":[41],"In":[42],"this":[43],"paper,":[44],"we":[45,63,92],"introduce":[46],"a":[47,57,65,99],"method":[48],"aligning":[50],"multimodal":[51,180],"images":[53],"language":[55,85],"using":[56,98],"single":[58,143],"LVLM,":[59],"dubbed":[60],"UniMed-LVLM.":[61],"Specifically,":[62],"devise":[64],"General":[66],"Projection":[67],"Module":[68],"(GPM)":[69],"by":[70,169],"integrating":[71],"multiple":[72,95],"projection":[74],"branches":[75],"implementing":[77],"dynamic":[78],"routing":[79],"between":[80],"encoder":[83],"decoder":[86],"within":[87],"LLaVA-Med":[89,116],"framework.":[90],"Subsequently,":[91],"progressively":[93],"align":[94],"modalities":[97,149],"Parameter-Efficient":[100],"Fine-Tuning":[101],"(PEFT)":[102],"technique":[103],"known":[104],"as":[105],"Low-Rank":[106],"Adaptation":[107],"(LoRA).":[108],"The":[109],"model":[110,144],"is":[111,145],"initially":[112],"trained":[113],"then":[119],"fine-tuned":[120,146],"four":[122,173],"datasets:":[126],"PathVqa,":[127],"Slake,":[128],"VqaRad,":[129],"Fitzpatrick17k,":[131],"enabling":[132],"simultaneous":[134],"of":[136],"radiology,":[137],"pathology,":[138],"dermatology":[140],"images.":[141,182],"A":[142],"three":[148],"across":[150,171],"these":[151],"datasets":[152],"evaluated":[154],"all":[156],"test":[157],"sets.":[158],"Experimental":[159],"results":[160],"show":[161],"that":[162],"UniMed-LVLM":[163],"improves":[164],"average":[166],"evaluation":[167],"score":[168],"1.88%":[170],"datasets,":[174],"validating":[175],"its":[176],"effectiveness":[177],"handling":[179]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}