{"id":"https://openalex.org/W7161264523","doi":"https://doi.org/10.48550/arxiv.2605.14200","title":"How to Scale Mixture-of-Experts: From muP to the Maximally Scale-Stable Parameterization","display_name":"How to Scale Mixture-of-Experts: From muP to the Maximally Scale-Stable Parameterization","publication_year":2026,"publication_date":"2026-05-13","ids":{"openalex":"https://openalex.org/W7161264523","doi":"https://doi.org/10.48550/arxiv.2605.14200"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.14200","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14200","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.14200","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064168352","display_name":"Leena Chennuru Vankadara","orcid":"https://orcid.org/0000-0002-3810-840X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vankadara, Leena Chennuru","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087139769","display_name":"Moritz Haas","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haas, Moritz","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001112727","display_name":"Luke Hayward","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hayward, Luke","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051291800","display_name":"Sebastian Bordt","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bordt, Sebastian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136244508","display_name":"Alessandro Breccia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Breccia, Alessandro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.708899974822998,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.708899974822998,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.10019999742507935,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.016200000420212746,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.7307999730110168},{"id":"https://openalex.org/keywords/monotonic-function","display_name":"Monotonic function","score":0.7123000025749207},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5687000155448914},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.49459999799728394},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.4828000068664551},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4781000018119812},{"id":"https://openalex.org/keywords/limit","display_name":"Limit (mathematics)","score":0.46299999952316284},{"id":"https://openalex.org/keywords/observable","display_name":"Observable","score":0.4424000084400177}],"concepts":[{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.7307999730110168},{"id":"https://openalex.org/C72169020","wikidata":"https://www.wikidata.org/wiki/Q194404","display_name":"Monotonic function","level":2,"score":0.7123000025749207},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5687000155448914},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.49459999799728394},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.4828000068664551},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4781000018119812},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.46299999952316284},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4542999863624573},{"id":"https://openalex.org/C32848918","wikidata":"https://www.wikidata.org/wiki/Q845789","display_name":"Observable","level":2,"score":0.4424000084400177},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.43720000982284546},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.43070000410079956},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.4275999963283539},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.3433000147342682},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.32899999618530273},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.3231000006198883},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.30230000615119934},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.2953000068664551},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.29280000925064087},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2797999978065491},{"id":"https://openalex.org/C79379906","wikidata":"https://www.wikidata.org/wiki/Q3174497","display_name":"Dynamical systems theory","level":2,"score":0.27799999713897705},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.27219998836517334},{"id":"https://openalex.org/C73000952","wikidata":"https://www.wikidata.org/wiki/Q17007827","display_name":"Discretization","level":2,"score":0.2578999996185303},{"id":"https://openalex.org/C149810388","wikidata":"https://www.wikidata.org/wiki/Q5374873","display_name":"Emulation","level":2,"score":0.2572000026702881},{"id":"https://openalex.org/C63002673","wikidata":"https://www.wikidata.org/wiki/Q2260590","display_name":"Scoring rule","level":2,"score":0.25270000100135803}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.14200","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14200","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.14200","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14200","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"frontier":[1],"large":[2],"language":[3],"models":[4],"predominantly":[5],"rely":[6],"on":[7],"Mixture-of-Experts":[8],"(MoE)":[9],"architectures.":[10],"Despite":[11],"empirical":[12],"progress,":[13],"there":[14],"is":[15],"still":[16],"no":[17],"principled":[18,53],"understanding":[19],"of":[20,33,79,98,103,166,248,255],"how":[21],"hyperparameters":[22],"should":[23],"scale":[24,145,172,227],"with":[25,144,226,231],"network":[26],"width":[27,30],"$N$,":[28],"expert":[29,251],"$N_e$,":[31],"number":[32,254],"experts":[34],"$M$,":[35],"sparsity":[36],"$K$,":[37],"and":[38,45,74,83,123,188,195,223,253],"depth":[39],"$L$":[40],"to":[41,154],"ensure":[42],"both":[43,186],"stability":[44],"optimal":[46],"performance":[47],"at":[48],"scale.":[49],"We":[50,130,150],"take":[51],"a":[52,90,107,163,180,210,238,246],"step":[54],"toward":[55],"resolving":[56],"this":[57,114,176],"gap":[58],"by":[59,175],"analyzing":[60],"three":[61,192],"different":[62],"scaling":[63,78,193,240],"regimes:":[64],"(I)":[65],"co-scaling":[66,70],"$N\\asymp":[67,71],"N_e$,":[68],"(II)":[69],"M\\asymp":[72],"K$,":[73],"(III)":[75],"full":[76],"proportional":[77],"$N,":[80],"N_e,":[81],"M$,":[82],"$K$.":[84],"For":[85],"each":[86],"regime,":[87],"we":[88,116,169,178],"develop":[89],"novel":[91],"Dynamical":[92],"Mean":[93],"Field":[94],"Theory":[95],"(DMFT)":[96],"description":[97],"the":[99,118,134,158,197,205],"limiting":[100,199],"training":[101],"dynamics":[102,200],"MoEs":[104],"that":[105,133,168,216],"provides":[106],"formal":[108],"foundation":[109],"for":[110,121,185,242],"our":[111],"analysis.":[112,213],"Within":[113],"framework,":[115],"derive":[117,179],"unique":[119],"parameterization":[120],"SGD":[122,187],"Adam":[124,189],"satisfying":[125],"all":[126,191],"maximal-update":[127],"($\u03bc$)":[128],"desiderata.":[129],"then":[131],"show":[132],"resulting":[135],"$\u03bc$P":[136,206],"prescription":[137,241],"does":[138],"not":[139],"reliably":[140],"induce":[141],"monotonic":[142,224],"improvement":[143,225],"or":[146],"robust":[147],"learning-rate":[148],"transfer.":[149],"trace":[151],"these":[152,235],"pathologies":[153],"scale-dependent":[155],"observables":[156],"in":[157,190],"aggregation":[159],"dynamics,":[160],"which":[161],"motivates":[162],"refined":[164],"set":[165],"desiderata":[167],"term":[170],"maximal":[171],"stability.":[173],"Guided":[174],"principle,":[177],"Maximally":[181],"Scale-Stable":[182],"Parameterization":[183],"(MSSP)":[184],"regimes,":[194],"characterize":[196],"corresponding":[198],"-":[201,208],"qualitatively":[202],"distinct":[203],"from":[204],"limit":[207],"through":[209],"separate":[211],"DMFT":[212],"Experiments":[214],"verify":[215],"MSSP":[217],"robustly":[218],"recovers":[219],"learning":[220],"rate":[221],"transfer":[222],"across":[228],"regimes.":[229],"Combined":[230],"existing":[232],"depth-scaling":[233],"theory,":[234],"results":[236],"provide":[237],"complete":[239],"MoE":[243],"architectures":[244],"as":[245],"function":[247],"width,":[249,252],"depth,":[250],"experts.":[256]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-16T00:00:00"}