{"id":"https://openalex.org/W7203637376","doi":"https://doi.org/10.48550/arxiv.2608.14492","title":"Approximate Muon with low-rank adapters","display_name":"Approximate Muon with low-rank adapters","publication_year":2026,"publication_date":"2026-08-14","ids":{"openalex":"https://openalex.org/W7203637376","doi":"https://doi.org/10.48550/arxiv.2608.14492"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.14492","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.14492","pdf_url":"https://arxiv.org/pdf/2608.14492","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.14492","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028634767","display_name":"Ben Anson","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Anson, Ben","raw_affiliation_strings":["University of Bristol School of Mathematics"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Bristol School of Mathematics","institution_ids":["https://openalex.org/I36234482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147335019","display_name":"Conor Houghton","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Houghton, Conor","raw_affiliation_strings":["University of Bristol School of Engineering Mathematics and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Bristol School of Engineering Mathematics and Technology","institution_ids":["https://openalex.org/I36234482"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056339931","display_name":"Edward Milsom","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I51601045","display_name":"University of Bath","ror":"https://ror.org/002h8g185","country_code":"GB","type":"education","lineage":["https://openalex.org/I51601045"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Milsom, Edward","raw_affiliation_strings":["University of Bath Department of Computer Science"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Bath Department of Computer Science","institution_ids":["https://openalex.org/I51601045"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.7426999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.7426999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.16300000250339508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0215000007301569,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/muon-optimizer","display_name":"Muon optimizer","score":0.9900000095367432},{"id":"https://openalex.org/keywords/low-rank-adapters","display_name":"low-rank adapters","score":0.7910000085830688},{"id":"https://openalex.org/keywords/parameter-efficient-fine-tuning","display_name":"parameter-efficient fine-tuning","score":0.9789999723434448},{"id":"https://openalex.org/keywords/lora","display_name":"LoRa","score":0.8859999775886536},{"id":"https://openalex.org/keywords/linearization","display_name":"linearization","score":0.7540000081062317},{"id":"https://openalex.org/keywords/least-squares","display_name":"least squares","score":0.8140000104904175}],"concepts":[{"id":"https://openalex.org/C205334942","wikidata":"https://www.wikidata.org/wiki/Q3151","display_name":"Muon","level":2,"score":0.8335000276565552},{"id":"https://openalex.org/C11210021","wikidata":"https://www.wikidata.org/wiki/Q1520713","display_name":"Linearization","level":3,"score":0.633400022983551},{"id":"https://openalex.org/C124681953","wikidata":"https://www.wikidata.org/wiki/Q339062","display_name":"Decomposition","level":2,"score":0.4090000092983246},{"id":"https://openalex.org/C109214941","wikidata":"https://www.wikidata.org/wiki/Q18334","display_name":"Particle physics","level":1,"score":0.4018999934196472},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3822999894618988},{"id":"https://openalex.org/C106152062","wikidata":"https://www.wikidata.org/wiki/Q6937066","display_name":"Muon collider","level":4,"score":0.3817000091075897},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.3691999912261963},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.36160001158714294},{"id":"https://openalex.org/C136119220","wikidata":"https://www.wikidata.org/wiki/Q1000660","display_name":"Algebra over a field","level":2,"score":0.35760000348091125},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.29589998722076416}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.14492","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.14492","pdf_url":"https://arxiv.org/pdf/2608.14492","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.14492","doi":"https://doi.org/10.48550/arxiv.2608.14492","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.14492","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.14492","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.14492","pdf_url":"https://arxiv.org/pdf/2608.14492","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7203637376.pdf","grobid_xml":"https://content.openalex.org/works/W7203637376.grobid-xml"},"referenced_works_count":32,"referenced_works":["https://openalex.org/W2794325560","https://openalex.org/W2890894339","https://openalex.org/W2898695519","https://openalex.org/W2908510526","https://openalex.org/W2989588035","https://openalex.org/W2996908057","https://openalex.org/W3083410900","https://openalex.org/W3168867926","https://openalex.org/W3177813494","https://openalex.org/W3195140297","https://openalex.org/W4318908031","https://openalex.org/W4384263459","https://openalex.org/W4391591020","https://openalex.org/W4392575839","https://openalex.org/W4396815229","https://openalex.org/W4401306886","https://openalex.org/W4403815243","https://openalex.org/W4404334675","https://openalex.org/W4405655184","https://openalex.org/W4414803876","https://openalex.org/W4414846785","https://openalex.org/W4415089271","https://openalex.org/W4415798387","https://openalex.org/W4417511325","https://openalex.org/W7128376444","https://openalex.org/W7129002753","https://openalex.org/W7133235158","https://openalex.org/W7160889841","https://openalex.org/W7164564891","https://openalex.org/W7165385035","https://openalex.org/W7169847197","https://openalex.org/W7171552453"],"related_works":["https://openalex.org/W7164564891","https://openalex.org/W7154373369","https://openalex.org/W4415089271","https://openalex.org/W7109148841","https://openalex.org/W7169847197","https://openalex.org/W7147057124","https://openalex.org/W7133235158","https://openalex.org/W7125628310","https://openalex.org/W7160889841","https://openalex.org/W4402671706"],"abstract_inverted_index":{"The":[0],"Muon":[1,38,69,124],"optimizer":[2],"shows":[3],"clear":[4],"benefits":[5],"versus":[6],"alternatives":[7],"when":[8],"pretraining":[9,111],"neural":[10],"networks.":[11],"However,":[12],"it":[13,40],"is":[14,25,41],"used":[15],"less":[16],"frequently":[17],"for":[18,125],"parameter-efficient":[19],"fine-tuning":[20,127],"(PEFT).":[21],"One":[22],"potential":[23],"reason":[24],"that":[26,85,122],"the":[27,47,64,72],"most":[28],"common":[29],"PEFT":[30],"method,":[31,100],"LoRA,":[32],"does":[33],"not":[34,42],"naturally":[35],"combine":[36],"with":[37],"since":[39],"mathematically":[43],"possible":[44],"to":[45,66,92],"orthogonalize":[46],"weight":[48],"update":[49],"given":[50],"by":[51,62],"a":[52,67,109],"low-rank":[53,73,126],"parameterization.":[54],"In":[55],"this":[56,60],"paper,":[57],"we":[58,119],"address":[59],"issue":[61],"approximating":[63],"solution":[65],"relaxed":[68],"objective":[70],"in":[71],"setting":[74],"via":[75],"linearization":[76],"and":[77,108,117],"then":[78],"least-squares.":[79],"We":[80],"provide":[81],"an":[82],"efficient":[83],"implementation":[84],"uses":[86],"matmul":[87],"operations":[88],"only,":[89],"as":[90],"opposed":[91],"more":[93],"complex":[94],"linear":[95],"algebra":[96],"decomposition":[97],"routines.":[98],"Our":[99],"sMuon":[101],"(small":[102],"Muon),":[103],"performs":[104],"favourably":[105],"across":[106],"SFT":[107],"ReLoRA":[110],"experiment.":[112],"While":[113],"results":[114],"are":[115],"model-":[116],"eval-dependent,":[118],"find":[120],"overall":[121],"using":[123],"provides":[128],"moderate":[129],"performance":[130],"improvements.":[131]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-18T00:00:00"}