{"id":"https://openalex.org/W4380926296","doi":"https://doi.org/10.1145/3581791.3596844","title":"Harmony: Heterogeneous Multi-Modal Federated Learning through Disentangled Model Training","display_name":"Harmony: Heterogeneous Multi-Modal Federated Learning through Disentangled Model Training","publication_year":2023,"publication_date":"2023-06-16","ids":{"openalex":"https://openalex.org/W4380926296","doi":"https://doi.org/10.1145/3581791.3596844"},"language":"en","primary_location":{"id":"doi:10.1145/3581791.3596844","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3581791.3596844","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st Annual International Conference on Mobile Systems, Applications and Services","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5087839973","display_name":"Xiaomin Ouyang","orcid":null},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":true,"raw_author_name":"Xiaomin Ouyang","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0003-0710-0963","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005853578","display_name":"Zhiyuan Xie","orcid":null},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhiyuan Xie","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0002-9982-0222","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023775015","display_name":"Heming Fu","orcid":"https://orcid.org/0000-0002-2221-0143"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Heming Fu","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0002-2221-0143","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042789366","display_name":"Sitong Cheng","orcid":"https://orcid.org/0009-0006-6045-4612"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Sitong Cheng","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0009-0006-6045-4612","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057459100","display_name":"L J Pan","orcid":null},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Li Pan","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0002-3607-9393","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049748462","display_name":"Neiwen Ling","orcid":"https://orcid.org/0000-0003-2072-1502"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Neiwen Ling","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0003-2072-1502","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072780737","display_name":"Guoliang Xing","orcid":"https://orcid.org/0000-0003-1772-7751"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Guoliang Xing","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0003-1772-7751","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047215778","display_name":"Jiayu Zhou","orcid":"https://orcid.org/0000-0003-4336-6777"},"institutions":[{"id":"https://openalex.org/I87216513","display_name":"Michigan State University","ror":"https://ror.org/05hs6h993","country_code":"US","type":"education","lineage":["https://openalex.org/I87216513"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiayu Zhou","raw_affiliation_strings":["Michigan State University, East Lansing, United States of America"],"raw_orcid":"https://orcid.org/0000-0003-4336-6777","affiliations":[{"raw_affiliation_string":"Michigan State University, East Lansing, United States of America","institution_ids":["https://openalex.org/I87216513"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062346297","display_name":"Jianwei Huang","orcid":"https://orcid.org/0000-0001-6631-1096"},"institutions":[{"id":"https://openalex.org/I4210104064","display_name":"Shenzhen Academy of Robotics","ror":"https://ror.org/01h027j09","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210104064"]},{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianwei Huang","raw_affiliation_strings":["Shenzhen Institute of Artificial Intelligence and Robotics for Society, Shen Zhen, China","The Chinese University of Hong Kong, Shenzhen, Shen Zhen, China","The Chinese University of Hong Kong, Shenzhen, Shen Zhen, China Shenzhen Institute of Artificial Intelligence and Robotics for Society, Shen Zhen, China"],"raw_orcid":"https://orcid.org/0000-0001-6631-1096","affiliations":[{"raw_affiliation_string":"Shenzhen Institute of Artificial Intelligence and Robotics for Society, Shen Zhen, China","institution_ids":["https://openalex.org/I4210104064"]},{"raw_affiliation_string":"The Chinese University of Hong Kong, Shenzhen, Shen Zhen, China","institution_ids":["https://openalex.org/I4210116924"]},{"raw_affiliation_string":"The Chinese University of Hong Kong, Shenzhen, Shen Zhen, China Shenzhen Institute of Artificial Intelligence and Robotics for Society, Shen Zhen, China","institution_ids":["https://openalex.org/I4210116924"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5087839973"],"corresponding_institution_ids":["https://openalex.org/I177725633"],"apc_list":null,"apc_paid":null,"fwci":8.35,"has_fulltext":false,"cited_by_count":49,"citation_normalized_percentile":{"value":0.9823395,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"530","last_page":"543"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10326","display_name":"Indoor and Outdoor Localization Technologies","score":0.9452000260353088,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9429000020027161,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8077359199523926},{"id":"https://openalex.org/keywords/testbed","display_name":"Testbed","score":0.773316502571106},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.7117323279380798},{"id":"https://openalex.org/keywords/harmony","display_name":"Harmony (color)","score":0.6284223198890686},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.46758389472961426},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.44192254543304443},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.422435998916626},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.3610232472419739},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.15106093883514404}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8077359199523926},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.773316502571106},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.7117323279380798},{"id":"https://openalex.org/C2776453491","wikidata":"https://www.wikidata.org/wiki/Q5659234","display_name":"Harmony (color)","level":2,"score":0.6284223198890686},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.46758389472961426},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44192254543304443},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.422435998916626},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3610232472419739},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.15106093883514404},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3581791.3596844","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3581791.3596844","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st Annual International Conference on Mobile Systems, Applications and Services","raw_type":"proceedings-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-141698","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-141698","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference paper"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17","score":0.4099999964237213}],"awards":[{"id":"https://openalex.org/G8222963720","display_name":null,"funder_award_id":"62271434","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":47,"referenced_works":["https://openalex.org/W2050006425","https://openalex.org/W2061090644","https://openalex.org/W2086509056","https://openalex.org/W2099242680","https://openalex.org/W2145343602","https://openalex.org/W2341323008","https://openalex.org/W2553915786","https://openalex.org/W2604268533","https://openalex.org/W2736307756","https://openalex.org/W2741295496","https://openalex.org/W2751594996","https://openalex.org/W2767290858","https://openalex.org/W2791951661","https://openalex.org/W2835064718","https://openalex.org/W2896422817","https://openalex.org/W2950283932","https://openalex.org/W2965144482","https://openalex.org/W2972136518","https://openalex.org/W3021897136","https://openalex.org/W3029790089","https://openalex.org/W3035333188","https://openalex.org/W3038022836","https://openalex.org/W3080935714","https://openalex.org/W3094342158","https://openalex.org/W3104219743","https://openalex.org/W3109995084","https://openalex.org/W3136562528","https://openalex.org/W3174401204","https://openalex.org/W3177183117","https://openalex.org/W3209905867","https://openalex.org/W3210053704","https://openalex.org/W3210103168","https://openalex.org/W3211771663","https://openalex.org/W3211781253","https://openalex.org/W3212167271","https://openalex.org/W3213319487","https://openalex.org/W4206936477","https://openalex.org/W4283032505","https://openalex.org/W4283208915","https://openalex.org/W4289841367","https://openalex.org/W4290945297","https://openalex.org/W4301104990","https://openalex.org/W4306178637","https://openalex.org/W4306179740","https://openalex.org/W4306179760","https://openalex.org/W4306250089","https://openalex.org/W6759238902"],"related_works":["https://openalex.org/W2883256816","https://openalex.org/W2171408034","https://openalex.org/W3003320923","https://openalex.org/W2106140982","https://openalex.org/W2152313554","https://openalex.org/W2064303750","https://openalex.org/W4285042611","https://openalex.org/W1509300825","https://openalex.org/W3092582874","https://openalex.org/W627697492"],"abstract_inverted_index":{"Multi-modal":[0],"sensing":[1],"systems":[2],"are":[3],"increasingly":[4],"prevalent":[5],"in":[6,86,107,114,142],"real-world":[7,137],"applications":[8,165],"such":[9,57],"as":[10,58],"health":[11],"monitoring":[12],"and":[13,61,95,110,127,158,178],"autonomous":[14],"driving.":[15],"Most":[16],"multi-modal":[17,54,77,83,138],"learning":[18,34,40,94],"approaches":[19,45],"need":[20],"to":[21,30,172,181],"access":[22],"users'":[23,31],"raw":[24],"data,":[25],"which":[26],"poses":[27],"significant":[28],"concerns":[29],"privacy.":[32],"Federated":[33],"(FL)":[35],"provides":[36],"a":[37,72,87,101,136],"privacy-aware":[38],"distributed":[39],"framework.":[41],"However,":[42],"current":[43],"FL":[44,55,109],"have":[46],"not":[47],"addressed":[48],"the":[49,82,120,143,156],"unique":[50],"challenges":[51],"of":[52,145,163],"heterogeneous":[53,76],"systems,":[56],"modality":[59,112],"heterogeneity":[60],"significantly":[62],"longer":[63],"training":[64,85,183],"delay.":[65,184],"In":[66],"this":[67],"paper,":[68],"we":[69],"propose":[70],"Harmony,":[71],"new":[73],"system":[74,130],"for":[75,149],"federated":[78,93,96,115],"learning.":[79,98],"Harmony":[80,118,134,168],"disentangles":[81],"network":[84],"novel":[88,102],"two-stage":[89],"framework,":[90],"namely":[91],"modality-wise":[92,108],"fusion":[97,116],"By":[99],"integrating":[100],"balance-aware":[103],"resource":[104],"allocation":[105],"mechanism":[106],"exploiting":[111],"biases":[113],"learning,":[117],"improves":[119],"model":[121],"accuracy":[122,174],"under":[123],"non-i.i.d.":[124],"data":[125],"distributions":[126],"speeds":[128],"up":[129,171,180],"convergence.":[131],"We":[132],"implemented":[133],"on":[135,155],"sensor":[139],"testbed":[140,157],"deployed":[141],"homes":[144],"16":[146],"elderly":[147],"subjects":[148],"Alzheimer's":[150],"Disease":[151],"monitoring.":[152],"Our":[153],"evaluation":[154],"three":[159],"large-scale":[160],"public":[161],"datasets":[162],"different":[164],"show":[166],"that,":[167],"outperforms":[169],"by":[170],"46.35%":[173],"over":[175],"state-of-the-art":[176],"baselines":[177],"saves":[179],"30%":[182]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":23},{"year":2024,"cited_by_count":22},{"year":2023,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}