{"id":"https://openalex.org/W7162435015","doi":"https://doi.org/10.48550/arxiv.2605.24770","title":"Muon in Vision Transformers: Optimizer-Recipe Interactions and Gradient Spectra","display_name":"Muon in Vision Transformers: Optimizer-Recipe Interactions and Gradient Spectra","publication_year":2026,"publication_date":"2026-05-23","ids":{"openalex":"https://openalex.org/W7162435015","doi":"https://doi.org/10.48550/arxiv.2605.24770"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.24770","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.24770","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.24770","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013739420","display_name":"Ben S. Southworth","orcid":"https://orcid.org/0000-0002-0283-4928"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Southworth, Ben S.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137008654","display_name":"Shuai Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Shuai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137005967","display_name":"Daniel McBride","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"McBride, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134539867","display_name":"Eric C. Cyr","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cyr, Eric C.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137081748","display_name":"Stephen Thomas","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Thomas, Stephen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11216","display_name":"Radiation Detection and Scintillator Technologies","score":0.08320000022649765,"subfield":{"id":"https://openalex.org/subfields/3108","display_name":"Radiation"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11216","display_name":"Radiation Detection and Scintillator Technologies","score":0.08320000022649765,"subfield":{"id":"https://openalex.org/subfields/3108","display_name":"Radiation"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.08110000193119049,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.07119999825954437,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/muon","display_name":"Muon","score":0.9333000183105469},{"id":"https://openalex.org/keywords/autoencoder","display_name":"Autoencoder","score":0.40400001406669617},{"id":"https://openalex.org/keywords/spectral-line","display_name":"Spectral line","score":0.40389999747276306},{"id":"https://openalex.org/keywords/segmentation","display_name":"Segmentation","score":0.303600013256073},{"id":"https://openalex.org/keywords/high-energy","display_name":"High energy","score":0.30160000920295715},{"id":"https://openalex.org/keywords/approx","display_name":"Approx","score":0.29429998993873596},{"id":"https://openalex.org/keywords/spectral-shape-analysis","display_name":"Spectral shape analysis","score":0.2858000099658966}],"concepts":[{"id":"https://openalex.org/C205334942","wikidata":"https://www.wikidata.org/wiki/Q3151","display_name":"Muon","level":2,"score":0.9333000183105469},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.6796000003814697},{"id":"https://openalex.org/C109214941","wikidata":"https://www.wikidata.org/wiki/Q18334","display_name":"Particle physics","level":1,"score":0.45170000195503235},{"id":"https://openalex.org/C101738243","wikidata":"https://www.wikidata.org/wiki/Q786435","display_name":"Autoencoder","level":3,"score":0.40400001406669617},{"id":"https://openalex.org/C4839761","wikidata":"https://www.wikidata.org/wiki/Q212111","display_name":"Spectral line","level":2,"score":0.40389999747276306},{"id":"https://openalex.org/C185544564","wikidata":"https://www.wikidata.org/wiki/Q81197","display_name":"Nuclear physics","level":1,"score":0.3610999882221222},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.3605000078678131},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.303600013256073},{"id":"https://openalex.org/C30475298","wikidata":"https://www.wikidata.org/wiki/Q909554","display_name":"Computational physics","level":1,"score":0.30300000309944153},{"id":"https://openalex.org/C2985973956","wikidata":"https://www.wikidata.org/wiki/Q1617745","display_name":"High energy","level":2,"score":0.30160000920295715},{"id":"https://openalex.org/C2777894999","wikidata":"https://www.wikidata.org/wiki/Q4781758","display_name":"Approx","level":2,"score":0.29429998993873596},{"id":"https://openalex.org/C152822103","wikidata":"https://www.wikidata.org/wiki/Q7575207","display_name":"Spectral shape analysis","level":3,"score":0.2858000099658966},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.2847999930381775},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2842999994754791},{"id":"https://openalex.org/C12426560","wikidata":"https://www.wikidata.org/wiki/Q189569","display_name":"Basis (linear algebra)","level":2,"score":0.27630001306533813},{"id":"https://openalex.org/C186370098","wikidata":"https://www.wikidata.org/wiki/Q442787","display_name":"Energy (signal processing)","level":2,"score":0.27570000290870667},{"id":"https://openalex.org/C168110828","wikidata":"https://www.wikidata.org/wiki/Q1331626","display_name":"Spectral density","level":2,"score":0.27379998564720154},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.2662999927997589},{"id":"https://openalex.org/C38858127","wikidata":"https://www.wikidata.org/wiki/Q5441228","display_name":"Feed forward","level":2,"score":0.2653000056743622},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.2581999897956848},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.257999986410141},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.25589999556541443}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.24770","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.24770","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.24770","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.24770","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Muon":[0,29,54,73,102,152,160,175,228],"is":[1,22,163,199],"a":[2,110,126,147,172,189,195],"recently":[3],"developed":[4],"matrix-aware":[5],"optimizer":[6],"that":[7],"has":[8],"shown":[9],"strong":[10],"results":[11],"in":[12,18,117,121,137,146,161,181,209,217,231],"transformer":[13],"training,":[14,32],"but":[15],"its":[16,185],"behavior":[17],"vision":[19,43],"transformers":[20],"(ViTs)":[21],"not":[23],"yet":[24],"well":[25],"understood.":[26],"We":[27,213],"study":[28],"for":[30,201],"ViT":[31],"largely":[33],"on":[34,62,220],"ImageNet-100":[35],"and":[36,49,52,81,114,206,223],"Pl@ntNet-300K,":[37],"comparing":[38],"against":[39],"AdamW":[40,78,141,178,230],"under":[41],"standard":[42],"recipes":[44],"involving":[45],"mixup,":[46],"cutmix,":[47],"smoothing,":[48],"random":[50],"augmentation":[51,84,108,129],"erasing.":[53],"consistently":[55],"outperforms":[56,229],"AdamW,":[57],"with":[58],"especially":[59],"large":[60],"gains":[61,68],"long-tailed":[63],"Pl@ntNet":[64],"macro":[65],"top-1.":[66],"These":[67],"are":[69],"also":[70],"recipe-dependent,":[71],"where":[72,140,184,227],"benefits":[74],"much":[75,148],"more":[76,157],"than":[77],"from":[79,177],"advanced":[80],"significant":[82],"data":[83,107],"techniques.":[85],"To":[86],"understand":[87],"this":[88],"interaction,":[89],"we":[90],"analyze":[91],"the":[92,99,131],"singular-value":[93],"structure":[94],"of":[95,188],"matrix":[96],"gradients":[97,186],"throughout":[98],"ViT.":[100],"Within":[101,193],"training":[103,197,218],"runs,":[104],"removing":[105],"heavy":[106],"induces":[109],"late-training":[111],"spectral":[112,191,204],"concentration":[113,205],"mode":[115,207],"collapse":[116,208],"gradient":[118,142],"matrices,":[119],"primarily":[120],"deep":[122,210],"MLP-down":[123],"blocks.":[124,212],"Under":[125,171],"fixed":[127,173],"\"full\"":[128],"recipe,":[130,174],"clearest":[132],"Muon-AdamW":[133],"contrast":[134],"appears":[135],"instead":[136],"QKV":[138],"gradients,":[139],"energy":[143,154],"remains":[144],"concentrated":[145],"narrower":[149],"basis":[150],"while":[151],"spreads":[153],"across":[155],"substantially":[156],"singular":[158],"modes.":[159],"ViTs":[162,219],"therefore":[164],"best":[165],"understood":[166],"as":[167],"an":[168],"optimizer-recipe":[169],"interaction.":[170],"differs":[176],"most":[179],"clearly":[180],"attention":[182],"projections,":[183],"consist":[187],"broader":[190],"basis.":[192],"Muon,":[194],"full":[196],"recipe":[198],"important":[200],"preventing":[202],"late":[203],"feedforward":[211],"further":[214],"demonstrate":[215],"efficacy":[216],"image":[221],"segmentation":[222],"masked":[224],"autoencoder":[225],"models,":[226],"all":[232],"settings":[233],"considered.":[234]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-27T00:00:00"}