{"id":"https://openalex.org/W7133221758","doi":"https://doi.org/10.1109/tpds.2026.3668887","title":"Pro-Prophet: A Systematic Load Balancing Method for Efficient Parallel Training of Large-Scale MoE Models","display_name":"Pro-Prophet: A Systematic Load Balancing Method for Efficient Parallel Training of Large-Scale MoE Models","publication_year":2026,"publication_date":"2026-03-02","ids":{"openalex":"https://openalex.org/W7133221758","doi":"https://doi.org/10.1109/tpds.2026.3668887"},"language":null,"primary_location":{"id":"doi:10.1109/tpds.2026.3668887","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2026.3668887","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Wei Wang","orcid":"https://orcid.org/0009-0008-9377-1678"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Wang","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0009-0008-9377-1678","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104293176","display_name":"Zhiquan Lai","orcid":null},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiquan Lai","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-3458-4732","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127864882","display_name":"Dongsheng Li","orcid":null},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongsheng Li","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0001-9743-2034","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106623069","display_name":"Shengwei Li","orcid":"https://orcid.org/0000-0002-7419-1511"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shengwei Li","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-7419-1511","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Weijie Liu","orcid":"https://orcid.org/0000-0002-0354-9467"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weijie Liu","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-0354-9467","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074708780","display_name":"Keshi Ge","orcid":"https://orcid.org/0000-0002-0669-6892"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Keshi Ge","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-0669-6892","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101929743","display_name":"Ao Shen","orcid":"https://orcid.org/0009-0009-9186-6400"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ao Shen","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070555846","display_name":"Huayou Su","orcid":"https://orcid.org/0000-0002-3587-0917"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huayou Su","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-3587-0917","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I170215575"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.20664541,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"37","issue":"5","first_page":"1121","last_page":"1135"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.14650000631809235,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.14650000631809235,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.05739999935030937,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.05609999969601631,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.743399977684021},{"id":"https://openalex.org/keywords/planner","display_name":"Planner","score":0.6115999817848206},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.5917999744415283},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5861999988555908},{"id":"https://openalex.org/keywords/load-balancing","display_name":"Load balancing (electrical power)","score":0.48750001192092896},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.4717999994754791},{"id":"https://openalex.org/keywords/load-management","display_name":"Load management","score":0.4189999997615814},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.3560999929904938}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8709999918937683},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.743399977684021},{"id":"https://openalex.org/C2776999362","wikidata":"https://www.wikidata.org/wiki/Q2349274","display_name":"Planner","level":2,"score":0.6115999817848206},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5917999744415283},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5861999988555908},{"id":"https://openalex.org/C138959212","wikidata":"https://www.wikidata.org/wiki/Q1806783","display_name":"Load balancing (electrical power)","level":3,"score":0.48750001192092896},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.4717999994754791},{"id":"https://openalex.org/C2779370713","wikidata":"https://www.wikidata.org/wiki/Q357554","display_name":"Load management","level":2,"score":0.4189999997615814},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4113999903202057},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38749998807907104},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37880000472068787},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.3560999929904938},{"id":"https://openalex.org/C163175372","wikidata":"https://www.wikidata.org/wiki/Q3339222","display_name":"Linear model","level":2,"score":0.3465000092983246},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.3386000096797943},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.3303000032901764},{"id":"https://openalex.org/C41045048","wikidata":"https://www.wikidata.org/wiki/Q202843","display_name":"Linear programming","level":2,"score":0.3147999942302704},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3082999885082245},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.29910001158714294},{"id":"https://openalex.org/C155032097","wikidata":"https://www.wikidata.org/wiki/Q798503","display_name":"Backpropagation","level":3,"score":0.29580000042915344},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.2921000123023987},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2671000063419342},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.26649999618530273},{"id":"https://openalex.org/C2778915421","wikidata":"https://www.wikidata.org/wiki/Q3643177","display_name":"Performance improvement","level":2,"score":0.26100000739097595},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.2508000135421753}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2026.3668887","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2026.3668887","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4412212870","display_name":null,"funder_award_id":"62025208","funder_id":"https://openalex.org/F4320327720","funder_display_name":"Foundation for Innovative Research Groups of the National Natural Science Foundation of China"},{"id":"https://openalex.org/G7870814796","display_name":null,"funder_award_id":"62421002","funder_id":"https://openalex.org/F4320327720","funder_display_name":"Foundation for Innovative Research Groups of the National Natural Science Foundation of China"},{"id":"https://openalex.org/G8203333444","display_name":null,"funder_award_id":"62302512","funder_id":"https://openalex.org/F4320327720","funder_display_name":"Foundation for Innovative Research Groups of the National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320327720","display_name":"Foundation for Innovative Research Groups of the National Natural Science Foundation of China","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":32,"referenced_works":["https://openalex.org/W1814428676","https://openalex.org/W2138690674","https://openalex.org/W2152556309","https://openalex.org/W2963786636","https://openalex.org/W3129831491","https://openalex.org/W3202134982","https://openalex.org/W4220838824","https://openalex.org/W4220967350","https://openalex.org/W4283815817","https://openalex.org/W4289828103","https://openalex.org/W4312910503","https://openalex.org/W4312983671","https://openalex.org/W4327694855","https://openalex.org/W4364382874","https://openalex.org/W4376652719","https://openalex.org/W4380759903","https://openalex.org/W4385571616","https://openalex.org/W4386260498","https://openalex.org/W4386396242","https://openalex.org/W4386709668","https://openalex.org/W4386768656","https://openalex.org/W4388855670","https://openalex.org/W4394923534","https://openalex.org/W4396817294","https://openalex.org/W4399073645","https://openalex.org/W4399452094","https://openalex.org/W4401508784","https://openalex.org/W4405440297","https://openalex.org/W4408232561","https://openalex.org/W4408858386","https://openalex.org/W4416310693","https://openalex.org/W7133212984"],"related_works":[],"abstract_inverted_index":{"The":[0,13,230],"size":[1,22],"of":[2,36,60,103,135,145,177,197,260],"deep":[3],"learning":[4],"models":[5,47,63,242,269],"has":[6,39,255],"been":[7,86],"increasing":[8],"to":[9,48,88,107,151,164,237,244,262,271,274],"enhance":[10],"model":[11,21,29,104],"quality.":[12],"linear":[14],"increase":[15],"in":[16,224],"training":[17,25,59,144,159,189,218],"computation":[18,54],"budgets":[19],"with":[20,51],"means":[23],"that":[24,233],"an":[26],"extremely":[27],"large-scale":[28,61,146],"is":[30],"exceedingly":[31],"time-consuming.":[32],"Recently,":[33],"the":[34,90,152,171,192,201,205,210,217],"Mixture":[35],"Experts":[37],"(MoE)":[38],"drawn":[40],"significant":[41],"attention":[42,96],"as":[43],"it":[44],"can":[45],"scale":[46],"extra-large":[49],"sizes":[50],"a":[52,69,128,136,139,175,185,257,275],"near-stable":[53],"budget.":[55],"However,":[56,110],"inefficient":[57],"distributed":[58],"MoE":[62,147,228,247,268],"hinders":[64],"their":[65,98],"broader":[66],"application.":[67],"Specifically,":[68],"considerable":[70],"dynamic":[71,153],"load":[72,154],"imbalance":[73],"occurs":[74],"among":[75],"devices":[76],"during":[77],"training,":[78],"significantly":[79],"reducing":[80],"throughput.":[81,219],"Several":[82],"load-balancing":[83,130,179,258,277],"works":[84],"have":[85,157],"proposed":[87],"address":[89,123],"challenge.":[91],"System-level":[92],"solutions":[93],"draw":[94],"more":[95],"for":[97,141,184,188],"hardware":[99],"affinity":[100],"and":[101,118,138,161,181,199,212,227,251,264],"non-disruption":[102],"convergence":[105],"compared":[106,243,273],"algorithm-level":[108],"ones.":[109],"they":[111],"are":[112],"troubled":[113],"by":[114],"high":[115],"communication":[116,169,198],"costs":[117],"poor":[119],"communication-computation":[120],"overlap.":[121],"To":[122,149],"these":[124],"challenges,":[125],"we":[126,156],"propose":[127],"systematic":[129],"method,":[131],"Pro-Prophet,":[132],"which":[133],"consists":[134],"planner":[137,173],"scheduler":[140,203],"efficient":[142],"parallel":[143],"models.":[148,229],"adapt":[150],"imbalance,":[155],"profiled":[158],"statistics":[160,211],"utilized":[162],"them":[163],"design":[165],"Pro-Prophet.":[166],"For":[167,194],"lower":[168],"volume,":[170],"Pro-Prophet":[172,202,234,254],"determines":[174],"series":[176],"lightweight":[178],"strategies":[180],"efficiently":[182],"searches":[183],"communication-efficient":[186],"one":[187],"based":[190,208],"on":[191,209,240,266],"statistics.":[193],"sufficient":[195],"overlapping":[196],"computation,":[200],"schedules":[204],"data-dependent":[206],"operations":[207],"operation":[213],"characteristics,":[214],"further":[215],"improving":[216],"We":[220],"conduct":[221],"extensive":[222],"experiments":[223],"various":[225],"clusters":[226],"results":[231],"indicate":[232],"achieves":[235],"up":[236,261,270],"2.66x":[238],"speedup":[239],"MoE-GPT":[241],"two":[245],"popular":[246],"frameworks,":[248],"namely":[249],"Deepspeed-MoE":[250],"FasterMoE.":[252,279],"Furthermore,":[253],"demonstrated":[256],"improvement":[259],"11.01x":[263],"speedups":[265],"modern":[267],"1.22x":[272],"representative":[276],"work,":[278]},"counts_by_year":[],"updated_date":"2026-03-25T13:04:00.132906","created_date":"2026-03-03T00:00:00"}