{"id":"https://openalex.org/W4413500911","doi":"https://doi.org/10.1109/ton.2025.3585359","title":"Communication-Efficient Sparsely-Activated Model Training via Sequence Migration and Token Condensation","display_name":"Communication-Efficient Sparsely-Activated Model Training via Sequence Migration and Token Condensation","publication_year":2025,"publication_date":"2025-07-08","ids":{"openalex":"https://openalex.org/W4413500911","doi":"https://doi.org/10.1109/ton.2025.3585359"},"language":"en","primary_location":{"id":"doi:10.1109/ton.2025.3585359","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ton.2025.3585359","pdf_url":null,"source":{"id":"https://openalex.org/S5407042750","display_name":"IEEE Transactions on Networking","issn_l":"2998-4157","issn":["2998-4157"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Networking","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088461411","display_name":"Fahao Chen","orcid":"https://orcid.org/0000-0002-4345-1296"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fahao Chen","raw_affiliation_strings":["School of Artificial Intelligence, Shandong University, Jinan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Shandong University, Jinan, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066098631","display_name":"Peng Li","orcid":"https://orcid.org/0000-0002-5303-0700"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peng Li","raw_affiliation_strings":["School of Cyber Science and Engineering, Xi’an Jiaotong University, Xi’an, China"],"raw_orcid":"https://orcid.org/0000-0002-5303-0700","affiliations":[{"raw_affiliation_string":"School of Cyber Science and Engineering, Xi’an Jiaotong University, Xi’an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075795671","display_name":"Zicong Hong","orcid":"https://orcid.org/0000-0001-5689-382X"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zicong Hong","raw_affiliation_strings":["Department of Computer Science and Engineering, The Hong Kong University of Science and Technology, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0001-5689-382X","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, The Hong Kong University of Science and Technology, Hong Kong, China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101149934","display_name":"Zhou Su","orcid":"https://orcid.org/0000-0002-6518-3130"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhou Su","raw_affiliation_strings":["School of Cyber Science and Engineering, Xi’an Jiaotong University, Xi’an, China"],"raw_orcid":"https://orcid.org/0000-0002-6518-3130","affiliations":[{"raw_affiliation_string":"School of Cyber Science and Engineering, Xi’an Jiaotong University, Xi’an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043464306","display_name":"Song Guo","orcid":"https://orcid.org/0000-0001-9831-2202"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Song Guo","raw_affiliation_strings":["Department of Computer Science and Engineering, The Hong Kong University of Science and Technology, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0001-9831-2202","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, The Hong Kong University of Science and Technology, Hong Kong, China","institution_ids":["https://openalex.org/I200769079"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.6138,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.95630064,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":"33","issue":"6","first_page":"2869","last_page":"2880"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9879000186920166,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9879000186920166,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9689000248908997,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9325000047683716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.7155529260635376},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.7135423421859741},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.6714627742767334},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5459228754043579},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.4767635464668274},{"id":"https://openalex.org/keywords/token-passing","display_name":"Token passing","score":0.4177553057670593},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.17014548182487488},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.12305110692977905},{"id":"https://openalex.org/keywords/genetics","display_name":"Genetics","score":0.059338003396987915}],"concepts":[{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.7155529260635376},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.7135423421859741},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6714627742767334},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5459228754043579},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.4767635464668274},{"id":"https://openalex.org/C115067241","wikidata":"https://www.wikidata.org/wiki/Q1639854","display_name":"Token passing","level":3,"score":0.4177553057670593},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.17014548182487488},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.12305110692977905},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.059338003396987915},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/ton.2025.3585359","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ton.2025.3585359","pdf_url":null,"source":{"id":"https://openalex.org/S5407042750","display_name":"IEEE Transactions on Networking","issn_l":"2998-4157","issn":["2998-4157"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Networking","raw_type":"journal-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-169069","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-169069","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10","score":0.6899999976158142}],"awards":[{"id":"https://openalex.org/G7935549449","display_name":null,"funder_award_id":"62471383","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W2789202651","https://openalex.org/W2933138175","https://openalex.org/W2950856799","https://openalex.org/W2963748441","https://openalex.org/W2964110616","https://openalex.org/W2989743967","https://openalex.org/W3080189354","https://openalex.org/W3173909755","https://openalex.org/W3213120752","https://openalex.org/W4214686755","https://openalex.org/W4220838824","https://openalex.org/W4220967350","https://openalex.org/W4280612849","https://openalex.org/W4289828103","https://openalex.org/W4320067926","https://openalex.org/W4321636575","https://openalex.org/W4360831795","https://openalex.org/W4364382874","https://openalex.org/W4384705378","https://openalex.org/W4385245566","https://openalex.org/W4385567528","https://openalex.org/W4386260498","https://openalex.org/W4386396242","https://openalex.org/W4389609877","https://openalex.org/W4394923534","https://openalex.org/W4401508090","https://openalex.org/W4402671950","https://openalex.org/W4404782031"],"related_works":["https://openalex.org/W2765256135","https://openalex.org/W2074350650","https://openalex.org/W2540135243","https://openalex.org/W2016681143","https://openalex.org/W2348435129","https://openalex.org/W2013402399","https://openalex.org/W2397526281","https://openalex.org/W2057608425","https://openalex.org/W2136545404","https://openalex.org/W2106477326"],"abstract_inverted_index":{"Mixture-of-Experts":[0],"(MoE)":[1],"is":[2,104],"an":[3],"emerging":[4],"technique":[5],"for":[6],"scaling":[7],"large":[8],"models":[9,14],"with":[10,22,132],"sparse":[11],"activation.":[12],"MoE":[13,29,129,193],"are":[15,31],"typically":[16],"trained":[17],"in":[18,27],"a":[19,112,121,126,177,185],"distributed":[20,32,128],"manner":[21],"anexpert":[23],"parallelismscheme,":[24],"where":[25],"experts":[26,74,151],"each":[28],"layer":[30],"across":[33],"multiple":[34],"GPUs.":[35,153],"However,":[36],"the":[37,43,49,59,77],"default":[38],"expert":[39,60,87,116],"parallelism":[40,84],"suffers":[41],"from":[42],"heavy":[44,142],"network":[45,78],"burden":[46],"due":[47],"to":[48,67,75,100,106,140,191],"all-to-all":[50],"intermediate":[51,69],"data":[52,70],"exchange":[53],"among":[54,138],"GPUs":[55,139,147],"before":[56],"and":[57,89,148,163,172],"after":[58],"run.":[61],"Some":[62],"existing":[63,96],"works":[64,97],"have":[65],"proposed":[66],"reduce":[68,76,107],"exchanges":[71],"by":[72,124],"transferring":[73],"loads,":[79],"however,":[80],"which":[81],"would":[82],"decrease":[83],"level":[85],"of":[86,95,115,179,187],"execution":[88],"make":[90],"computation":[91],"inefficient.":[92],"The":[93],"weaknesses":[94],"motivate":[98],"us":[99],"explore":[101],"whether":[102],"it":[103],"possible":[105],"inter-GPU":[108],"traffic":[109],"while":[110],"maintaining":[111],"high":[113],"degree":[114],"parallelism.":[117],"This":[118],"paper":[119],"gives":[120],"positive":[122],"response":[123],"presentingLuffy,":[125],"communication-efficient":[127],"training":[130,194],"system":[131],"two":[133],"new":[134],"techniques.":[135],"First,Luffymigrates":[136],"sequences":[137],"hide":[141],"token":[143,157],"pulling":[144],"paths":[145],"within":[146],"avoid":[149],"copying":[150],"over":[152],"Second,":[154],"we":[155],"propose":[156],"condensation":[158],"that":[159],"identifies":[160],"similar":[161],"tokens":[162],"then":[164],"eliminates":[165],"redundant":[166],"transmissions.":[167],"We":[168],"implementLuffybased":[169],"on":[170,176],"PyTorch":[171],"evaluate":[173],"its":[174],"performance":[175],"testbed":[178],"16":[180],"V100":[181],"GPUs.Luffysystem":[182],"can":[183],"achieve":[184],"speedup":[186],"up":[188],"to$2.73\\times":[189],"$compared":[190],"state-of-the-art":[192],"systems.":[195]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":2}],"updated_date":"2026-08-12T21:12:35.861297","created_date":"2025-10-10T00:00:00"}