{"id":"https://openalex.org/W7160944534","doi":"https://doi.org/10.48550/arxiv.2605.08933","title":"When and Why Grouping Attention Heads Accelerates Muon Optimization","display_name":"When and Why Grouping Attention Heads Accelerates Muon Optimization","publication_year":2026,"publication_date":"2026-05-09","ids":{"openalex":"https://openalex.org/W7160944534","doi":"https://doi.org/10.48550/arxiv.2605.08933"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.08933","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08933","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.08933","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125548744","display_name":"Hongtao Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Hongtao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101918392","display_name":"Wenjie Zhou","orcid":"https://orcid.org/0000-0002-8385-7194"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Wenjie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135916355","display_name":"Wei Chen","orcid":"https://orcid.org/0009-0005-1124-8217"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135937537","display_name":"Xueqi Cheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Xueqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10048","display_name":"Particle physics theoretical and experimental studies","score":0.24070000648498535,"subfield":{"id":"https://openalex.org/subfields/3106","display_name":"Nuclear and High Energy Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10048","display_name":"Particle physics theoretical and experimental studies","score":0.24070000648498535,"subfield":{"id":"https://openalex.org/subfields/3106","display_name":"Nuclear and High Energy Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12579","display_name":"Muon and positron interactions and applications","score":0.1143999993801117,"subfield":{"id":"https://openalex.org/subfields/2211","display_name":"Mechanics of Materials"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11044","display_name":"Particle Detector Development and Performance","score":0.09939999878406525,"subfield":{"id":"https://openalex.org/subfields/3106","display_name":"Nuclear and High Energy Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/muon","display_name":"Muon","score":0.8924999833106995},{"id":"https://openalex.org/keywords/granularity","display_name":"Granularity","score":0.843500018119812},{"id":"https://openalex.org/keywords/norm","display_name":"Norm (philosophy)","score":0.4120999872684479},{"id":"https://openalex.org/keywords/matrix","display_name":"Matrix (chemical analysis)","score":0.3889999985694885},{"id":"https://openalex.org/keywords/head","display_name":"Head (geology)","score":0.3328999876976013}],"concepts":[{"id":"https://openalex.org/C205334942","wikidata":"https://www.wikidata.org/wiki/Q3151","display_name":"Muon","level":2,"score":0.8924999833106995},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.843500018119812},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5001999735832214},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.4311000108718872},{"id":"https://openalex.org/C191795146","wikidata":"https://www.wikidata.org/wiki/Q3878446","display_name":"Norm (philosophy)","level":2,"score":0.4120999872684479},{"id":"https://openalex.org/C109214941","wikidata":"https://www.wikidata.org/wiki/Q18334","display_name":"Particle physics","level":1,"score":0.3971000015735626},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.3889999985694885},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3497999906539917},{"id":"https://openalex.org/C2780312720","wikidata":"https://www.wikidata.org/wiki/Q5689100","display_name":"Head (geology)","level":2,"score":0.3328999876976013},{"id":"https://openalex.org/C185544564","wikidata":"https://www.wikidata.org/wiki/Q81197","display_name":"Nuclear physics","level":1,"score":0.33079999685287476},{"id":"https://openalex.org/C2776637919","wikidata":"https://www.wikidata.org/wiki/Q624380","display_name":"Descent (aeronautics)","level":2,"score":0.3248000144958496},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.30559998750686646},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.29319998621940613},{"id":"https://openalex.org/C2779072382","wikidata":"https://www.wikidata.org/wiki/Q2937690","display_name":"Muon capture","level":3,"score":0.27160000801086426},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2606000006198883},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.25040000677108765}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.08933","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08933","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.08933","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08933","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Muon":[0,22,50,117],"orthogonalizes":[1],"matrix":[2],"updates,":[3],"but":[4],"multi-head":[5],"attention":[6,29],"naturally":[7],"operates":[8],"at":[9],"the":[10,18,27,60,68],"level":[11],"of":[12,20],"heads.":[13],"This":[14],"granularity":[15],"mismatch":[16],"raises":[17],"question":[19,42],"whether":[21],"should":[23],"be":[24],"applied":[25],"to":[26,31,35],"full":[28],"projection,":[30],"individual":[32],"heads,":[33],"or":[34],"intermediate":[36],"head":[37,94],"groups.":[38],"We":[39],"study":[40],"this":[41,86],"through":[43],"a":[44,57],"one-step":[45],"descent":[46],"comparison":[47],"between":[48,59],"full-matrix":[49,79],"and":[51,67,97,118],"group-wise":[52,65,82],"Muon.":[53],"Our":[54],"analysis":[55],"reveals":[56],"trade-off":[58],"\\textbf{group-wise":[61],"whitening":[62,80],"gain}":[63],"from":[64],"updates":[66],"\\textbf{grouping-induced":[69],"norm":[70],"cost},":[71],"an":[72],"additional":[73],"update-norm":[74],"cost":[75],"caused":[76],"by":[77,85],"replacing":[78],"with":[81],"whitening.":[83],"Motivated":[84],"trade-off,":[87],"we":[88],"propose":[89],"\\textbf{Group":[90],"Muon},":[91],"which":[92],"treats":[93],"group":[95],"size":[96],"grouping":[98,110],"rule":[99],"as":[100],"optimizer":[101],"hyperparameters.":[102],"On":[103],"GPT-2":[104],"Small":[105],"trained":[106],"on":[107],"FineWeb,":[108],"appropriate":[109],"improves":[111],"validation":[112],"loss":[113],"over":[114],"both":[115],"full-QKV":[116],"fully":[119],"head-wise":[120],"MuonSplit.":[121]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-13T00:00:00"}