{"id":"https://openalex.org/W7123342257","doi":"https://doi.org/10.1109/tpami.2026.3651754","title":"Accelerated Optimization of Large Mixture-of-Experts Models by Density-Aware Multi-Stage Learning","display_name":"Accelerated Optimization of Large Mixture-of-Experts Models by Density-Aware Multi-Stage Learning","publication_year":2026,"publication_date":"2026-01-12","ids":{"openalex":"https://openalex.org/W7123342257","doi":"https://doi.org/10.1109/tpami.2026.3651754","pmid":"https://pubmed.ncbi.nlm.nih.gov/41525587"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2026.3651754","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2026.3651754","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5060309794","display_name":"J. Yu","orcid":null},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianxing Yu","raw_affiliation_strings":["School of Artificial Intelligence, Sun Yat-Sen University, Zhuhai, China"],"raw_orcid":"https://orcid.org/0000-0003-1340-3995","affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Sun Yat-Sen University, Zhuhai, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Haowei Jiang","orcid":"https://orcid.org/0000-0002-5619-5252"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haowei Jiang","raw_affiliation_strings":["School of Artificial Intelligence, Sun Yat-Sen University, Zhuhai, China"],"raw_orcid":"https://orcid.org/0000-0002-5619-5252","affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Sun Yat-Sen University, Zhuhai, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122876323","display_name":"Huaijie Zhu","orcid":null},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huaijie Zhu","raw_affiliation_strings":["School of Artificial Intelligence, Sun Yat-Sen University, Zhuhai, China"],"raw_orcid":"https://orcid.org/0000-0001-8263-9032","affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Sun Yat-Sen University, Zhuhai, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Wenqing Chen","orcid":"https://orcid.org/0000-0002-8739-2216"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenqing Chen","raw_affiliation_strings":["School of Software Engineering, Sun Yat-sen University, Zhuhai, China"],"raw_orcid":"https://orcid.org/0000-0002-8739-2216","affiliations":[{"raw_affiliation_string":"School of Software Engineering, Sun Yat-sen University, Zhuhai, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119027813","display_name":"Yanghui Rao","orcid":null},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanghui Rao","raw_affiliation_strings":["School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-1610-9599","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043729465","display_name":"Qinliang Su","orcid":"https://orcid.org/0000-0002-5903-2504"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qinliang Su","raw_affiliation_strings":["School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-5903-2504","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"last","author":{"id":null,"display_name":"Jian Yin","orcid":"https://orcid.org/0000-0002-1214-5384"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jian Yin","raw_affiliation_strings":["School of Artificial Intelligence, Sun Yat-Sen University, Zhuhai, China"],"raw_orcid":"https://orcid.org/0000-0002-1214-5384","affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Sun Yat-Sen University, Zhuhai, China","institution_ids":["https://openalex.org/I157773358"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I157773358"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.05105653,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"48","issue":"5","first_page":"5505","last_page":"5519"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.5041999816894531,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.5041999816894531,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.17960000038146973,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.03460000082850456,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.7426000237464905},{"id":"https://openalex.org/keywords/planner","display_name":"Planner","score":0.6029000282287598},{"id":"https://openalex.org/keywords/subject-matter-expert","display_name":"Subject-matter expert","score":0.5490999817848206},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5394999980926514},{"id":"https://openalex.org/keywords/domain-knowledge","display_name":"Domain knowledge","score":0.512499988079071},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5091000199317932},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5078999996185303},{"id":"https://openalex.org/keywords/expert-system","display_name":"Expert system","score":0.46860000491142273},{"id":"https://openalex.org/keywords/plan","display_name":"Plan (archaeology)","score":0.43970000743865967}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7896000146865845},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.7426000237464905},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6129999756813049},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6115999817848206},{"id":"https://openalex.org/C2776999362","wikidata":"https://www.wikidata.org/wiki/Q2349274","display_name":"Planner","level":2,"score":0.6029000282287598},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.5490999817848206},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5394999980926514},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.512499988079071},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5091000199317932},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5078999996185303},{"id":"https://openalex.org/C58328972","wikidata":"https://www.wikidata.org/wiki/Q184609","display_name":"Expert system","level":2,"score":0.46860000491142273},{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.43970000743865967},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.4350000023841858},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.42579999566078186},{"id":"https://openalex.org/C117896860","wikidata":"https://www.wikidata.org/wiki/Q11376","display_name":"Acceleration","level":2,"score":0.42289999127388},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.41850000619888306},{"id":"https://openalex.org/C34585555","wikidata":"https://www.wikidata.org/wiki/Q1368723","display_name":"Learning curve","level":2,"score":0.3596000075340271},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.3513999879360199},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3418000042438507},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.31940001249313354},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.3190999925136566},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.2874999940395355},{"id":"https://openalex.org/C17020691","wikidata":"https://www.wikidata.org/wiki/Q139677","display_name":"Operator (biology)","level":5,"score":0.2858000099658966},{"id":"https://openalex.org/C114073186","wikidata":"https://www.wikidata.org/wiki/Q2631895","display_name":"Automated planning and scheduling","level":2,"score":0.2806999981403351}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2026.3651754","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2026.3651754","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:41525587","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/41525587","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1232371104","display_name":null,"funder_award_id":"62276280","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2014871491","display_name":null,"funder_award_id":"2024A1515010253","funder_id":"https://openalex.org/F4320337111","funder_display_name":"Basic and Applied Basic Research Foundation of Guangdong Province"},{"id":"https://openalex.org/G380298712","display_name":null,"funder_award_id":"U22B2060","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4580034021","display_name":null,"funder_award_id":"62306344","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4915157000","display_name":null,"funder_award_id":"62372483","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5720687473","display_name":null,"funder_award_id":"62276279","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6139988440","display_name":null,"funder_award_id":"2024B1515020032","funder_id":"https://openalex.org/F4320337111","funder_display_name":"Basic and Applied Basic Research Foundation of Guangdong Province"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320337111","display_name":"Basic and Applied Basic Research Foundation of Guangdong Province","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"This":[0,122],"article":[1],"aims":[2],"to":[3,27,57,59,68,130,192,197,206,219,225,232,241,258,269,303,313],"speed":[4,110],"up":[5,191],"the":[6,13,47,185,203,208,212,220,227,238,243,248,254,271,277,291,306,315],"training":[7,134,144,160,228,245,283,330],"of":[8,23,184,247,317],"large":[9,29],"neural":[10],"networks":[11,80,90],"with":[12,43,81,180],"Mixture-of-Experts":[14],"(MoE)":[15],"structure.":[16],"Training":[17],"MoE":[18,86],"often":[19],"needs":[20],"a":[21,82,142,166,175,182,266,301],"lot":[22],"computing":[24],"resources":[25,48],"due":[26],"its":[28],"scale.":[30],"Traditional":[31],"acceleration":[32,145,331],"methods":[33],"either":[34],"degrade":[35],"prediction":[36],"performance":[37],"or":[38],"rely":[39],"on":[40,332],"dedicated":[41],"hardware":[42],"additional":[44],"resources,":[45],"but":[46,85],"are":[49,76,286,311],"usually":[50],"limited":[51],"in":[52,92,113,165,214],"real":[53],"applications.One":[54],"solution":[55],"is":[56],"resort":[58],"new":[60,143],"optimization":[61],"strategies,":[62],"such":[63],"as":[64,231],"learning":[65,98,107,152,307],"from":[66,284],"easy":[67],"hard":[69,127],"by":[70,157,211,296],"multiple":[71,88],"stages.":[72],"However,":[73],"existing":[74],"strategies":[75],"designed":[77],"mainly":[78],"for":[79,100,128,154,282],"serial":[83],"structure,":[84],"has":[87],"expert":[89,156,213,244],"working":[91],"parallel.":[93],"They":[94],"employ":[95],"an":[96,150,198],"identical":[97],"plan":[99,153],"all":[101],"experts,":[102],"ignoring":[103],"that":[104,178,323],"each":[105,155,215,252],"expert's":[106],"domain":[108],"and":[109,187,279],"differ,":[111],"resulting":[112],"some":[114,259],"experts":[115,129,221],"being":[116,120],"over-learned":[117],"while":[118],"others":[119],"under-learned.":[121],"mismatch":[123],"will":[124],"make":[125],"it":[126,190,195],"train":[131,270],"together,":[132],"harming":[133],"efficiency.":[135],"To":[136,289],"address":[137],"this":[138],"problem,":[139],"we":[140,172,236,299,324],"propose":[141],"framework.":[146],"It":[147,201],"can":[148,264,325],"customize":[149],"effective":[151],"considering":[158],"their":[159],"progress,":[161],"avoiding":[162],"blindly":[163],"searching":[164],"huge":[167],"parameter":[168],"space.":[169],"In":[170,251],"detail,":[171],"first":[173],"design":[174],"multi-stage":[176],"planner":[177],"starts":[179],"optimizing":[181],"subpart":[183],"network":[186,255,297],"then":[188],"scales":[189],"retrain":[193],"until":[194],"expands":[196],"entire":[199],"network.":[200],"uses":[202],"density":[204],"function":[205],"assess":[207],"knowledge":[209],"gained":[210],"stage,":[216,253],"giving":[217],"priority":[218],"who":[222],"learn":[223],"faster":[224],"increase":[226],"scale,":[229],"so":[230],"boost":[233],"convergence.":[234],"Afterward,":[235],"exploit":[237],"growth":[239],"operator":[240],"add":[242],"scale":[246],"next":[249,272],"stage.":[250],"would":[256],"converge":[257],"locally":[260],"optimal":[261],"values.":[262],"That":[263],"provide":[265],"better":[267],"initialization":[268],"stage":[273],"more":[274,327],"easily,":[275],"since":[276],"time":[278],"data":[280],"required":[281],"scratch":[285],"greatly":[287],"reduced.":[288],"alleviate":[290],"gradient":[292],"vanishing":[293],"problem":[294],"caused":[295],"growth,":[298],"develop":[300],"scheduler":[302],"dynamically":[304],"adjust":[305],"rate.":[308],"Extensive":[309],"experiments":[310],"conducted":[312],"validate":[314],"effectiveness":[316],"our":[318],"method.":[319],"The":[320],"results":[321],"show":[322],"obtain":[326],"than":[328],"25%":[329],"average.":[333]},"counts_by_year":[],"updated_date":"2026-04-04T06:10:10.580331","created_date":"2026-01-14T00:00:00"}