{"id":"https://openalex.org/W4413500911","doi":"https://doi.org/10.1109/ton.2025.3585359","title":"Communication-Efficient Sparsely-Activated Model Training via Sequence Migration and Token Condensation","display_name":"Communication-Efficient Sparsely-Activated Model Training via Sequence Migration and Token Condensation","publication_year":2025,"publication_date":"2025-07-08","ids":{"openalex":"https://openalex.org/W4413500911","doi":"https://doi.org/10.1109/ton.2025.3585359"},"language":"en","primary_location":{"id":"doi:10.1109/ton.2025.3585359","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ton.2025.3585359","pdf_url":null,"source":{"id":"https://openalex.org/S5407042750","display_name":"IEEE Transactions on Networking","issn_l":"2998-4157","issn":["2998-4157"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Networking","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088461411","display_name":"Fahao Chen","orcid":"https://orcid.org/0000-0002-4345-1296"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fahao Chen","raw_affiliation_strings":["School of Artificial Intelligence, Shandong University, Jinan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Shandong University, Jinan, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066098631","display_name":"Peng Li","orcid":"https://orcid.org/0000-0002-5303-0700"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peng Li","raw_affiliation_strings":["School of Cyber Science and Engineering, Xi&#x2019;an Jiaotong University, Xi&#x2019;an, China"],"raw_orcid":"https://orcid.org/0000-0002-5303-0700","affiliations":[{"raw_affiliation_string":"School of Cyber Science and Engineering, Xi&#x2019;an Jiaotong University, Xi&#x2019;an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075795671","display_name":"Zicong Hong","orcid":"https://orcid.org/0000-0001-5689-382X"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zicong Hong","raw_affiliation_strings":["Department of Computer Science and Engineering, The Hong Kong University of Science and Technology, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0001-5689-382X","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, The Hong Kong University of Science and Technology, Hong Kong, China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101149934","display_name":"Zhou Su","orcid":"https://orcid.org/0000-0002-6518-3130"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhou Su","raw_affiliation_strings":["School of Cyber Science and Engineering, Xi&#x2019;an Jiaotong University, Xi&#x2019;an, China"],"raw_orcid":"https://orcid.org/0000-0002-6518-3130","affiliations":[{"raw_affiliation_string":"School of Cyber Science and Engineering, Xi&#x2019;an Jiaotong University, Xi&#x2019;an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043464306","display_name":"Song Guo","orcid":"https://orcid.org/0000-0001-9831-2202"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Song Guo","raw_affiliation_strings":["Department of Computer Science and Engineering, The Hong Kong University of Science and Technology, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0001-9831-2202","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, The Hong Kong University of Science and Technology, Hong Kong, China","institution_ids":["https://openalex.org/I200769079"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.217,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.9529678,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":"33","issue":"6","first_page":"2869","last_page":"2880"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9879000186920166,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9879000186920166,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9689000248908997,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9325000047683716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.7155529260635376},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.7135423421859741},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.6714627742767334},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5459228754043579},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.4767635464668274},{"id":"https://openalex.org/keywords/token-passing","display_name":"Token passing","score":0.4177553057670593},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.17014548182487488},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.12305110692977905},{"id":"https://openalex.org/keywords/genetics","display_name":"Genetics","score":0.059338003396987915}],"concepts":[{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.7155529260635376},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.7135423421859741},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6714627742767334},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5459228754043579},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.4767635464668274},{"id":"https://openalex.org/C115067241","wikidata":"https://www.wikidata.org/wiki/Q1639854","display_name":"Token passing","level":3,"score":0.4177553057670593},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.17014548182487488},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.12305110692977905},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.059338003396987915},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/ton.2025.3585359","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ton.2025.3585359","pdf_url":null,"source":{"id":"https://openalex.org/S5407042750","display_name":"IEEE Transactions on Networking","issn_l":"2998-4157","issn":["2998-4157"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Networking","raw_type":"journal-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-169069","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-169069","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.6899999976158142,"display_name":"Reduced inequalities"}],"awards":[{"id":"https://openalex.org/G7935549449","display_name":null,"funder_award_id":"62471383","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W2789202651","https://openalex.org/W2933138175","https://openalex.org/W2950856799","https://openalex.org/W2963748441","https://openalex.org/W2964110616","https://openalex.org/W2989743967","https://openalex.org/W3080189354","https://openalex.org/W3173909755","https://openalex.org/W3213120752","https://openalex.org/W4214686755","https://openalex.org/W4220838824","https://openalex.org/W4220967350","https://openalex.org/W4280612849","https://openalex.org/W4289828103","https://openalex.org/W4320067926","https://openalex.org/W4321636575","https://openalex.org/W4360831795","https://openalex.org/W4364382874","https://openalex.org/W4384705378","https://openalex.org/W4385245566","https://openalex.org/W4385567528","https://openalex.org/W4386260498","https://openalex.org/W4386396242","https://openalex.org/W4389609877","https://openalex.org/W4394923534","https://openalex.org/W4401508090","https://openalex.org/W4402671950","https://openalex.org/W4404782031"],"related_works":["https://openalex.org/W2765256135","https://openalex.org/W2074350650","https://openalex.org/W2540135243","https://openalex.org/W2016681143","https://openalex.org/W2348435129","https://openalex.org/W2013402399","https://openalex.org/W2397526281","https://openalex.org/W2057608425","https://openalex.org/W2136545404","https://openalex.org/W2106477326"],"abstract_inverted_index":{"Mixture-of-Experts":[0],"(MoE)":[1],"is":[2,108],"an":[3,23],"emerging":[4],"technique":[5],"for":[6],"scaling":[7],"large":[8],"models":[9,14],"with":[10,22,139],"sparse":[11],"activation.":[12],"MoE":[13,33,136,219],"are":[15,35],"typically":[16],"trained":[17],"in":[18,31],"a":[19,116,125,133,192,204],"distributed":[20,36,135],"manner":[21],"<italic":[24],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[25,131,145,182,199,210],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">expert":[26],"parallelism</i>":[27],"scheme,":[28],"where":[29],"experts":[30,78,162],"each":[32],"layer":[34],"across":[37],"multiple":[38],"GPUs.":[39,164,197],"However,":[40],"the":[41,47,53,63,81],"default":[42],"expert":[43,64,91,120],"parallelism":[44,88],"suffers":[45],"from":[46],"heavy":[48,153],"network":[49,82],"burden":[50],"due":[51],"to":[52,71,79,104,110,151,208,217],"all-to-all":[54],"intermediate":[55,73],"data":[56,74],"exchange":[57],"among":[58,149],"GPUs":[59,150,158],"before":[60],"and":[61,93,159,174,187],"after":[62],"run.":[65],"Some":[66],"existing":[67,100],"works":[68,101],"have":[69],"proposed":[70],"reduce":[72,80,111],"exchanges":[75],"by":[76,128],"transferring":[77],"loads,":[83],"however,":[84],"which":[85],"would":[86],"decrease":[87],"level":[89],"of":[90,99,119,194,206],"execution":[92],"make":[94],"computation":[95],"inefficient.":[96],"The":[97],"weaknesses":[98],"motivate":[102],"us":[103],"explore":[105],"whether":[106],"it":[107],"possible":[109],"inter-GPU":[112],"traffic":[113],"while":[114],"maintaining":[115],"high":[117],"degree":[118],"parallelism.":[121],"This":[122],"paper":[123],"gives":[124],"positive":[126],"response":[127],"presenting":[129],"<sc":[130,144,181,198],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Luffy</small>,":[132],"communication-efficient":[134],"training":[137,220],"system":[138,201],"two":[140],"new":[141],"techniques.":[142],"First,":[143],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Luffy</small>":[146,183,200],"migrates":[147],"sequences":[148],"hide":[152],"token":[154,168],"pulling":[155],"paths":[156],"within":[157],"avoid":[160],"copying":[161],"over":[163],"Second,":[165],"we":[166],"propose":[167],"condensation":[169],"that":[170],"identifies":[171],"similar":[172],"tokens":[173],"then":[175],"eliminates":[176],"redundant":[177],"transmissions.":[178],"We":[179],"implement":[180],"based":[184],"on":[185,191],"PyTorch":[186],"evaluate":[188],"its":[189],"performance":[190],"testbed":[193],"16":[195],"V100":[196],"can":[202],"achieve":[203],"speedup":[205],"up":[207],"<inline-formula":[209],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">":[211],"<tex-math":[212],"notation=\"LaTeX\">$2.73\\times":[213],"$</tex-math>":[214],"</inline-formula>":[215],"compared":[216],"state-of-the-art":[218],"systems.":[221]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}