{"id":"https://openalex.org/W4414198625","doi":"https://doi.org/10.1109/jbhi.2025.3609739","title":"Enhancing 3D Medical Image Understanding With Pretraining Aided by 2D Multimodal Large Language Models","display_name":"Enhancing 3D Medical Image Understanding With Pretraining Aided by 2D Multimodal Large Language Models","publication_year":2025,"publication_date":"2025-09-15","ids":{"openalex":"https://openalex.org/W4414198625","doi":"https://doi.org/10.1109/jbhi.2025.3609739","pmid":"https://pubmed.ncbi.nlm.nih.gov/40953412"},"language":"en","primary_location":{"id":"doi:10.1109/jbhi.2025.3609739","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jbhi.2025.3609739","pdf_url":null,"source":{"id":"https://openalex.org/S2495854775","display_name":"IEEE Journal of Biomedical and Health Informatics","issn_l":"2168-2194","issn":["2168-2194","2168-2208"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Biomedical and Health Informatics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048890580","display_name":"Qiuhui Chen","orcid":"https://orcid.org/0000-0002-1210-1306"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qiuhui Chen","raw_affiliation_strings":["School of Computer Science, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-1210-1306","affiliations":[{"raw_affiliation_string":"School of Computer Science, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052046224","display_name":"Xiaolei Yao","orcid":"https://orcid.org/0000-0001-8186-5097"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuancheng Yao","raw_affiliation_strings":["School of Computer Science, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074501277","display_name":"Huping Ye","orcid":"https://orcid.org/0000-0002-9114-205X"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huping Ye","raw_affiliation_strings":["School of Computer Science, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101931365","display_name":"Yi Hong","orcid":"https://orcid.org/0000-0002-9065-3691"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yi Hong","raw_affiliation_strings":["School of Computer Science, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-9065-3691","affiliations":[{"raw_affiliation_string":"School of Computer Science, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I183067930"],"apc_list":null,"apc_paid":null,"fwci":1.5311,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.84883145,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":97},"biblio":{"volume":"30","issue":"2","first_page":"1506","last_page":"1519"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9889000058174133,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9889000058174133,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10862","display_name":"AI in cancer detection","score":0.9175999760627747,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/medical-imaging","display_name":"Medical imaging","score":0.6299999952316284},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.6022999882698059},{"id":"https://openalex.org/keywords/image-segmentation","display_name":"Image segmentation","score":0.49959999322891235},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4526999890804291},{"id":"https://openalex.org/keywords/segmentation","display_name":"Segmentation","score":0.4472000002861023},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.4458000063896179},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.4417000114917755},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.39570000767707825},{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.38179999589920044},{"id":"https://openalex.org/keywords/convolution","display_name":"Convolution (computer science)","score":0.3776000142097473}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8330000042915344},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6449000239372253},{"id":"https://openalex.org/C31601959","wikidata":"https://www.wikidata.org/wiki/Q931309","display_name":"Medical imaging","level":2,"score":0.6299999952316284},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6022999882698059},{"id":"https://openalex.org/C124504099","wikidata":"https://www.wikidata.org/wiki/Q56933","display_name":"Image segmentation","level":3,"score":0.49959999322891235},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4526999890804291},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.4472000002861023},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.4458000063896179},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.4417000114917755},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.41359999775886536},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.39570000767707825},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.38179999589920044},{"id":"https://openalex.org/C45347329","wikidata":"https://www.wikidata.org/wiki/Q5166604","display_name":"Convolution (computer science)","level":3,"score":0.3776000142097473},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.37610000371932983},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.3594000041484833},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.3522000014781952},{"id":"https://openalex.org/C9417928","wikidata":"https://www.wikidata.org/wiki/Q1070689","display_name":"Image processing","level":3,"score":0.3481999933719635},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3474999964237213},{"id":"https://openalex.org/C534262118","wikidata":"https://www.wikidata.org/wiki/Q177719","display_name":"Medical diagnosis","level":2,"score":0.328000009059906},{"id":"https://openalex.org/C108882727","wikidata":"https://www.wikidata.org/wiki/Q2991685","display_name":"Solid modeling","level":2,"score":0.3109000027179718},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.30720001459121704},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.304500013589859},{"id":"https://openalex.org/C160633673","wikidata":"https://www.wikidata.org/wiki/Q355198","display_name":"Pixel","level":2,"score":0.30250000953674316},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.2879999876022339},{"id":"https://openalex.org/C2776674983","wikidata":"https://www.wikidata.org/wiki/Q545981","display_name":"Image editing","level":3,"score":0.2856999933719635},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.2768000066280365},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.27239999175071716},{"id":"https://openalex.org/C141379421","wikidata":"https://www.wikidata.org/wiki/Q6094427","display_name":"Iterative reconstruction","level":2,"score":0.2655999958515167},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.26330000162124634},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2628999948501587},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2612999975681305},{"id":"https://openalex.org/C106430172","wikidata":"https://www.wikidata.org/wiki/Q6002272","display_name":"Image restoration","level":4,"score":0.2517000138759613}],"mesh":[{"descriptor_ui":"D000069553","descriptor_name":"Supervised Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000069553","descriptor_name":"Supervised Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000077321","descriptor_name":"Deep Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000077321","descriptor_name":"Deep Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000098342","descriptor_name":"Large Language Models","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000098342","descriptor_name":"Large Language Models","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D007090","descriptor_name":"Image Interpretation, Computer-Assisted","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D007090","descriptor_name":"Image Interpretation, Computer-Assisted","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D008279","descriptor_name":"Magnetic Resonance Imaging","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008279","descriptor_name":"Magnetic Resonance Imaging","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D014057","descriptor_name":"Tomography, X-Ray Computed","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D014057","descriptor_name":"Tomography, X-Ray Computed","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016208","descriptor_name":"Databases, Factual","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016208","descriptor_name":"Databases, Factual","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D021621","descriptor_name":"Imaging, Three-Dimensional","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D021621","descriptor_name":"Imaging, Three-Dimensional","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true}],"locations_count":2,"locations":[{"id":"doi:10.1109/jbhi.2025.3609739","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jbhi.2025.3609739","pdf_url":null,"source":{"id":"https://openalex.org/S2495854775","display_name":"IEEE Journal of Biomedical and Health Informatics","issn_l":"2168-2194","issn":["2168-2194","2168-2208"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Biomedical and Health Informatics","raw_type":"journal-article"},{"id":"pmid:40953412","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40953412","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE journal of biomedical and health informatics","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6825432458","display_name":null,"funder_award_id":"62203303","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":56,"referenced_works":["https://openalex.org/W1901129140","https://openalex.org/W1973965874","https://openalex.org/W1985506636","https://openalex.org/W1988494453","https://openalex.org/W1996118086","https://openalex.org/W2036996178","https://openalex.org/W2124260444","https://openalex.org/W2150534249","https://openalex.org/W2155513557","https://openalex.org/W2291593693","https://openalex.org/W2724892359","https://openalex.org/W2896457183","https://openalex.org/W2902652978","https://openalex.org/W2910094941","https://openalex.org/W2964744899","https://openalex.org/W2979708377","https://openalex.org/W3002569343","https://openalex.org/W3007724151","https://openalex.org/W3014974815","https://openalex.org/W3035524453","https://openalex.org/W3090449556","https://openalex.org/W3096609285","https://openalex.org/W3171007011","https://openalex.org/W3172681723","https://openalex.org/W3183302989","https://openalex.org/W3201906559","https://openalex.org/W3204436700","https://openalex.org/W3208578242","https://openalex.org/W4206471589","https://openalex.org/W4212875960","https://openalex.org/W4221163766","https://openalex.org/W4304084190","https://openalex.org/W4312312750","https://openalex.org/W4312749457","https://openalex.org/W4312804044","https://openalex.org/W4313156423","https://openalex.org/W4313591097","https://openalex.org/W4317437782","https://openalex.org/W4319300504","https://openalex.org/W4384159609","https://openalex.org/W4385245566","https://openalex.org/W4385573131","https://openalex.org/W4386076097","https://openalex.org/W4386352556","https://openalex.org/W4387211502","https://openalex.org/W4387211540","https://openalex.org/W4387968041","https://openalex.org/W4387973771","https://openalex.org/W4390493241","https://openalex.org/W4390873519","https://openalex.org/W4399110454","https://openalex.org/W4399527342","https://openalex.org/W4402703096","https://openalex.org/W4405102058","https://openalex.org/W4405669265","https://openalex.org/W4413447271"],"related_works":[],"abstract_inverted_index":{"Understanding":[0],"3D":[1,13,53,66,109,151],"medical":[2,9,14,54,110,152],"image":[3,41,55,67,153],"volumes":[4],"is":[5],"critical":[6],"in":[7,29,98],"the":[8],"field,":[10],"yet":[11],"existing":[12,150],"convolution":[15],"and":[16,129,137],"transformer-based":[17],"self-supervised":[18],"learning":[19,112],"(SSL)":[20],"methods":[21],"often":[22],"lack":[23],"deep":[24],"semantic":[25],"comprehension.":[26],"Recent":[27],"advancements":[28],"multimodal":[30,108],"large":[31],"language":[32],"models":[33],"(MLLMs)":[34],"provide":[35],"a":[36,60,73,83,103],"promising":[37],"approach":[38],"to":[39,92],"enhance":[40],"understanding":[42,154],"through":[43],"text":[44],"descriptions.":[45],"To":[46],"leverage":[47],"these":[48],"2D":[49,70],"MLLMs":[50,71],"for":[51,106],"improved":[52],"understanding,":[56],"we":[57],"propose":[58],"Med3DInsight,":[59],"novel":[61],"pretraining":[62],"framework":[63],"that":[64],"integrates":[65],"encoders":[68],"with":[69,135],"via":[72],"specially":[74],"designed":[75],"plane-slice-aware":[76],"transformer":[77],"module.":[78],"Additionally,":[79],"our":[80,120],"model":[81],"employs":[82],"partial":[84],"optimal":[85],"transport":[86],"based":[87],"alignment,":[88],"demonstrating":[89],"greater":[90],"tolerance":[91],"noise":[93],"introduced":[94],"by":[95],"potential":[96],"noises":[97],"LLM-generated":[99],"content.":[100],"Med3DInsight":[101,144],"introduces":[102],"new":[104],"paradigm":[105],"scalable":[107],"representation":[111],"without":[113],"requiring":[114],"human":[115],"annotations.":[116],"Extensive":[117],"experiments":[118],"demonstrate":[119],"state-of-the-art":[121],"performance":[122],"on":[123],"two":[124],"downstream":[125],"tasks,":[126],"i.e.,":[127],"segmentation":[128],"classification,":[130],"across":[131],"various":[132],"public":[133],"datasets":[134],"CT":[136],"MRI":[138],"modalities,":[139],"outperforming":[140],"current":[141],"SSL":[142],"methods.":[143],"can":[145],"be":[146],"seamlessly":[147],"integrated":[148],"into":[149],"networks,":[155],"potentially":[156],"enhancing":[157],"their":[158],"performance.":[159]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-02-07T06:11:34.122080","created_date":"2025-10-10T00:00:00"}