{"id":"https://openalex.org/W4405812352","doi":"https://doi.org/10.1109/tmi.2024.3523378","title":"Tackling Modality-Heterogeneous Client Drift Holistically for Heterogeneous Multimodal Federated Learning","display_name":"Tackling Modality-Heterogeneous Client Drift Holistically for Heterogeneous Multimodal Federated Learning","publication_year":2024,"publication_date":"2024-12-26","ids":{"openalex":"https://openalex.org/W4405812352","doi":"https://doi.org/10.1109/tmi.2024.3523378","pmid":"https://pubmed.ncbi.nlm.nih.gov/40030772"},"language":"en","primary_location":{"id":"doi:10.1109/tmi.2024.3523378","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmi.2024.3523378","pdf_url":null,"source":{"id":"https://openalex.org/S58069681","display_name":"IEEE Transactions on Medical Imaging","issn_l":"0278-0062","issn":["0278-0062","1558-254X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Medical Imaging","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043165575","display_name":"Haoyue Song","orcid":null},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Haoyue Song","raw_affiliation_strings":["National Institute for Data Science in Health and Medicine, Xiamen University, Xiamen, China"],"raw_orcid":"https://orcid.org/0009-0007-9555-3482","affiliations":[{"raw_affiliation_string":"National Institute for Data Science in Health and Medicine, Xiamen University, Xiamen, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018181104","display_name":"Jiacheng Wang","orcid":"https://orcid.org/0000-0003-2595-265X"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiacheng Wang","raw_affiliation_strings":["Department of Computer Science, School of Informatics, Xiamen University, Xiamen, China"],"raw_orcid":"https://orcid.org/0000-0003-2595-265X","affiliations":[{"raw_affiliation_string":"Department of Computer Science, School of Informatics, Xiamen University, Xiamen, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009308493","display_name":"Jianjun Zhou","orcid":"https://orcid.org/0000-0003-4175-0513"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]},{"id":"https://openalex.org/I4210124907","display_name":"Zhongshan Hospital of Xiamen University","ror":"https://ror.org/02z125451","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210124907"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianjun Zhou","raw_affiliation_strings":["Department of Radiology, Zhongshan Hospital, Fudan University, Xiamen, China","Department of Radiology, Zhongshan Hospital (Xiamen), Fudan University, Xiamen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Radiology, Zhongshan Hospital, Fudan University, Xiamen, China","institution_ids":["https://openalex.org/I4210124907"]},{"raw_affiliation_string":"Department of Radiology, Zhongshan Hospital (Xiamen), Fudan University, Xiamen, China","institution_ids":["https://openalex.org/I4210124907","https://openalex.org/I24943067"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100613491","display_name":"Liansheng Wang","orcid":"https://orcid.org/0000-0002-2096-454X"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liansheng Wang","raw_affiliation_strings":["National Institute for Data Science in Health and Medicine and the Department of Computer Science, School of Informatics, Xiamen University, Xiamen, China","National Institute for Data Science in Health and Medicine, Xiamen University, Xiamen, China"],"raw_orcid":"https://orcid.org/0000-0002-2096-454X","affiliations":[{"raw_affiliation_string":"National Institute for Data Science in Health and Medicine and the Department of Computer Science, School of Informatics, Xiamen University, Xiamen, China","institution_ids":["https://openalex.org/I191208505"]},{"raw_affiliation_string":"National Institute for Data Science in Health and Medicine, Xiamen University, Xiamen, China","institution_ids":["https://openalex.org/I191208505"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5043165575"],"corresponding_institution_ids":["https://openalex.org/I191208505"],"apc_list":null,"apc_paid":null,"fwci":2.1884,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.89859627,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":100},"biblio":{"volume":"44","issue":"4","first_page":"1931","last_page":"1941"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9646999835968018,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9646999835968018,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11550","display_name":"Text and Document Classification Technologies","score":0.9442999958992004,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.7982367277145386},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6618911027908325},{"id":"https://openalex.org/keywords/multimodality","display_name":"Multimodality","score":0.4501422047615051},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3679850697517395},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.1542477011680603}],"concepts":[{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.7982367277145386},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6618911027908325},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.4501422047615051},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3679850697517395},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.1542477011680603}],"mesh":[{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000098407","descriptor_name":"Federated Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000098407","descriptor_name":"Federated Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000098407","descriptor_name":"Federated Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000098407","descriptor_name":"Federated Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000098407","descriptor_name":"Federated Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D001921","descriptor_name":"Brain","qualifier_ui":"Q000000981","qualifier_name":"diagnostic imaging","is_major_topic":false},{"descriptor_ui":"D001921","descriptor_name":"Brain","qualifier_ui":"Q000000981","qualifier_name":"diagnostic imaging","is_major_topic":false},{"descriptor_ui":"D001921","descriptor_name":"Brain","qualifier_ui":"Q000000981","qualifier_name":"diagnostic imaging","is_major_topic":false},{"descriptor_ui":"D001921","descriptor_name":"Brain","qualifier_ui":"Q000000981","qualifier_name":"diagnostic imaging","is_major_topic":false},{"descriptor_ui":"D001921","descriptor_name":"Brain","qualifier_ui":"Q000000981","qualifier_name":"diagnostic imaging","is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D007091","descriptor_name":"Image Processing, Computer-Assisted","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D007091","descriptor_name":"Image Processing, Computer-Assisted","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D007091","descriptor_name":"Image Processing, Computer-Assisted","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D007091","descriptor_name":"Image Processing, Computer-Assisted","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D007091","descriptor_name":"Image Processing, Computer-Assisted","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D008279","descriptor_name":"Magnetic Resonance Imaging","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D008279","descriptor_name":"Magnetic Resonance Imaging","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D008279","descriptor_name":"Magnetic Resonance Imaging","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D008279","descriptor_name":"Magnetic Resonance Imaging","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D008279","descriptor_name":"Magnetic Resonance Imaging","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D014057","descriptor_name":"Tomography, X-Ray Computed","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D014057","descriptor_name":"Tomography, X-Ray Computed","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D014057","descriptor_name":"Tomography, X-Ray Computed","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D014057","descriptor_name":"Tomography, X-Ray Computed","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D014057","descriptor_name":"Tomography, X-Ray Computed","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false}],"locations_count":2,"locations":[{"id":"doi:10.1109/tmi.2024.3523378","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmi.2024.3523378","pdf_url":null,"source":{"id":"https://openalex.org/S58069681","display_name":"IEEE Transactions on Medical Imaging","issn_l":"0278-0062","issn":["0278-0062","1558-254X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Medical Imaging","raw_type":"journal-article"},{"id":"pmid:40030772","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40030772","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on medical imaging","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1287903667","display_name":null,"funder_award_id":"2023J01005","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2807093373","display_name":null,"funder_award_id":"62371409","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4156630873","display_name":null,"funder_award_id":"STI2030-Major Projects2021ZD0201900","funder_id":"https://openalex.org/F4320321540","funder_display_name":"Ministry of Science and Technology of the People's Republic of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321540","display_name":"Ministry of Science and Technology of the People's Republic of China","ror":"https://ror.org/027s68j25"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":42,"referenced_works":["https://openalex.org/W1641498739","https://openalex.org/W2088139330","https://openalex.org/W2168161078","https://openalex.org/W2502805798","https://openalex.org/W2575552683","https://openalex.org/W2765961644","https://openalex.org/W2979640220","https://openalex.org/W3122794187","https://openalex.org/W3137996905","https://openalex.org/W3141518839","https://openalex.org/W3173432605","https://openalex.org/W3182158470","https://openalex.org/W3199488740","https://openalex.org/W3203934147","https://openalex.org/W4205335850","https://openalex.org/W4206279885","https://openalex.org/W4206936477","https://openalex.org/W4283367167","https://openalex.org/W4290945297","https://openalex.org/W4312869277","https://openalex.org/W4312950667","https://openalex.org/W4380926296","https://openalex.org/W6631190155","https://openalex.org/W6639103823","https://openalex.org/W6686207219","https://openalex.org/W6728757088","https://openalex.org/W6752667561","https://openalex.org/W6759238902","https://openalex.org/W6765817624","https://openalex.org/W6769105414","https://openalex.org/W6770590064","https://openalex.org/W6771904421","https://openalex.org/W6773817997","https://openalex.org/W6780224944","https://openalex.org/W6781318954","https://openalex.org/W6784239669","https://openalex.org/W6789305514","https://openalex.org/W6791444617","https://openalex.org/W6795843344","https://openalex.org/W6810433654","https://openalex.org/W6838655435","https://openalex.org/W6849911278"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4205899436","https://openalex.org/W2462138927","https://openalex.org/W3111427900","https://openalex.org/W4302009627","https://openalex.org/W4212781403","https://openalex.org/W4388989680","https://openalex.org/W2901011898"],"abstract_inverted_index":{"Multimodal":[0],"Federated":[1],"Learning":[2],"(MFL)":[3],"has":[4,40],"emerged":[5],"as":[6,173],"a":[7,32,105,110,128,165],"collaborative":[8],"paradigm":[9],"for":[10,82],"training":[11],"models":[12,212],"across":[13,51,184],"decentralized":[14],"devices,":[15],"harnessing":[16],"various":[17],"data":[18,26],"modalities":[19,50,187],"to":[20,37,91,96,104],"facilitate":[21],"effective":[22,131],"learning":[23],"while":[24,68,148,213],"respecting":[25],"ownership.":[27],"In":[28],"this":[29],"realm,":[30],"notably,":[31],"pivotal":[33],"shift":[34],"from":[35,117],"homogeneous":[36],"heterogeneous":[38,100,221,237],"MFL":[39,101,222,238],"taken":[41],"place.":[42],"While":[43],"the":[44,53,61,74,159,191],"former":[45],"assumes":[46],"uniformity":[47],"in":[48,63,226],"input":[49,186,218],"clients,":[52],"latter":[54],"accommodates":[55],"modality-incongruous":[56],"setups,":[57],"which":[58,171],"is":[59],"often":[60,87],"case":[62],"practical":[64],"situations.":[65],"For":[66],"example,":[67],"some":[69],"advanced":[70],"medical":[71,228],"institutions":[72],"have":[73],"luxury":[75],"of":[76],"utilizing":[77],"both":[78],"MRI":[79],"and":[80,144,168,188],"CT":[81,93],"disease":[83],"diagnosis,":[84],"remote":[85],"hospitals":[86],"find":[88],"themselves":[89],"constrained":[90],"employ":[92],"exclusively":[94],"due":[95],"its":[97,153],"cost-effectiveness.":[98],"Although":[99],"can":[102],"apply":[103],"broader":[106],"scenario,":[107],"it":[108],"introduces":[109],"new":[111],"challenge:":[112],"modality-heterogeneous":[113],"client":[114,205,211],"drift,":[115],"arising":[116],"diverse":[118,185],"modality-coupled":[119],"local":[120,134],"optimization.":[121],"To":[122,157],"address":[123],"this,":[124],"we":[125],"introduce":[126],"FedMM,":[127],"simple":[129],"yet":[130],"approach.":[132],"During":[133],"optimization,":[135],"FedMM":[136,163],"employs":[137],"modality":[138,155,160,195],"dropout,":[139],"randomly":[140],"masking":[141],"available":[142],"modalities,":[143,219],"promoting":[145],"weight":[146,179],"alignment":[147],"preserving":[149],"model":[150],"expressivity":[151],"on":[152],"original":[154],"combination.":[156],"enhance":[158],"dropout":[161,196],"process,":[162],"incorporates":[164],"task-specific":[166],"inter-":[167],"intra-modal":[169],"regularizer,":[170],"acts":[172],"an":[174],"additional":[175],"constraint,":[176],"forcing":[177],"that":[178],"distribution":[180],"remains":[181],"more":[182],"consistent":[183],"therefore":[189],"eases":[190],"optimization":[192],"process":[193],"with":[194],"enabled.":[197],"By":[198],"combining":[199],"them,":[200],"our":[201],"approach":[202],"holistically":[203],"addresses":[204],"drift.":[206],"It":[207],"fosters":[208],"convergence":[209],"among":[210],"considering":[214],"each":[215],"client's":[216],"unique":[217],"enhancing":[220],"performance.":[223],"Comprehensive":[224],"evaluations":[225],"three":[227],"image":[229],"segmentation":[230],"datasets":[231],"demonstrate":[232],"FedMM's":[233],"superiority":[234],"over":[235],"state-of-the-art":[236],"methods.":[239]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":2}],"updated_date":"2026-05-28T09:10:13.091523","created_date":"2025-10-10T00:00:00"}