{"id":"https://openalex.org/W7128468239","doi":"https://doi.org/10.48550/arxiv.2602.07425","title":"Sign-Based Optimizers Are Effective Under Heavy-Tailed Noise","display_name":"Sign-Based Optimizers Are Effective Under Heavy-Tailed Noise","publication_year":2026,"publication_date":"2026-02-07","ids":{"openalex":"https://openalex.org/W7128468239","doi":"https://doi.org/10.48550/arxiv.2602.07425"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2602.07425","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2602.07425","pdf_url":"https://arxiv.org/pdf/2602.07425","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2602.07425","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5109689425","display_name":"Dingzhi Yu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu, Dingzhi","raw_affiliation_strings":["School of Artificial Intelligence , Nanjing University","State Key Laboratory of Novel Software Technology , Nanjing University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence , Nanjing University","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"State Key Laboratory of Novel Software Technology , Nanjing University","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125541846","display_name":"Hongyi Tao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tao, Hongyi","raw_affiliation_strings":["State Key Laboratory of Novel Software Technology , Nanjing University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Novel Software Technology , Nanjing University","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088844680","display_name":"Yuanyu Wan","orcid":"https://orcid.org/0000-0002-3577-5186","observed_orcids":["https://orcid.org/0000-0002-3577-5186"]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wan, Yuanyu","raw_affiliation_strings":["School of Software Technology , Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Software Technology , Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042512583","display_name":"Luo Luo","orcid":"https://orcid.org/0009-0000-3641-8992","observed_orcids":["https://orcid.org/0009-0000-3641-8992","https://orcid.org/0000-0002-1943-2079"]},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Luo, Luo","raw_affiliation_strings":["School of Data Science , Fudan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Data Science , Fudan University","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5125545837","display_name":"Lijun Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhang, Lijun","raw_affiliation_strings":["School of Artificial Intelligence , Nanjing University","State Key Laboratory of Novel Software Technology , Nanjing University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence , Nanjing University","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"State Key Laboratory of Novel Software Technology , Nanjing University","institution_ids":["https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":16,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10963","display_name":"Advanced Optimization Algorithms Research","score":0.0,"subfield":{"id":"https://openalex.org/subfields/2612","display_name":"Numerical Analysis"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/heavy-tailed-noise","display_name":"heavy-tailed noise","score":0.8259999752044678},{"id":"https://openalex.org/keywords/lion-optimizer","display_name":"Lion optimizer","score":0.6539999842643738},{"id":"https://openalex.org/keywords/muon-optimizer","display_name":"Muon optimizer","score":0.9950000047683716},{"id":"https://openalex.org/keywords/signsgd","display_name":"signSGD","score":0.984000027179718},{"id":"https://openalex.org/keywords/convergence-rate","display_name":"convergence rate","score":0.7559999823570251},{"id":"https://openalex.org/keywords/large-language-model-pretraining","display_name":"large language model pretraining","score":0.7360000014305115},{"id":"https://openalex.org/keywords/matrix-optimization","display_name":"matrix optimization","score":0.6779999732971191},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"stochastic gradient descent","score":0.5950000286102295},{"id":"https://openalex.org/keywords/adaptive-gradient-methods","display_name":"adaptive gradient methods","score":0.5419999957084656}],"concepts":[{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.6905999779701233},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.628600001335144},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.5145000219345093},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.49239999055862427},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.47049999237060547},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.4652999937534332},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.45989999175071716},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.42969998717308044},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41760000586509705},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.35589998960494995},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33000001311302185},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.31139999628067017},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.2919999957084656},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.29170000553131104},{"id":"https://openalex.org/C29265498","wikidata":"https://www.wikidata.org/wiki/Q7047719","display_name":"Noise measurement","level":3,"score":0.2906000018119812},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2651999890804291},{"id":"https://openalex.org/C64812099","wikidata":"https://www.wikidata.org/wiki/Q176604","display_name":"Random matrix","level":3,"score":0.2605000138282776},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.2574000060558319}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2602.07425","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2602.07425","pdf_url":"https://arxiv.org/pdf/2602.07425","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2602.07425","doi":"https://doi.org/10.48550/arxiv.2602.07425","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.07425","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2602.07425","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2602.07425","pdf_url":"https://arxiv.org/pdf/2602.07425","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7128468239.pdf","grobid_xml":"https://content.openalex.org/works/W7128468239.grobid-xml"},"referenced_works_count":68,"referenced_works":["https://openalex.org/W1594567720","https://openalex.org/W2626325961","https://openalex.org/W2752366553","https://openalex.org/W2757910899","https://openalex.org/W2904243021","https://openalex.org/W2997976910","https://openalex.org/W2999589176","https://openalex.org/W3092641135","https://openalex.org/W4322718191","https://openalex.org/W4327810158","https://openalex.org/W4360826912","https://openalex.org/W4384918448","https://openalex.org/W4388090547","https://openalex.org/W4390041933","https://openalex.org/W4404400240","https://openalex.org/W4407212433","https://openalex.org/W4414281281","https://openalex.org/W4414486212","https://openalex.org/W4414634210","https://openalex.org/W4414757763","https://openalex.org/W4414803876","https://openalex.org/W4414846785","https://openalex.org/W4414944394","https://openalex.org/W4414977079","https://openalex.org/W4415035193","https://openalex.org/W4415068270","https://openalex.org/W4415141475","https://openalex.org/W4415145305","https://openalex.org/W4415194046","https://openalex.org/W4415250134","https://openalex.org/W4415330312","https://openalex.org/W4415336827","https://openalex.org/W4415938525","https://openalex.org/W4415966864","https://openalex.org/W4416075336","https://openalex.org/W4416101304","https://openalex.org/W4416157119","https://openalex.org/W4416330759","https://openalex.org/W4416373221","https://openalex.org/W4416445221","https://openalex.org/W4416522391","https://openalex.org/W4416613161","https://openalex.org/W4416692582","https://openalex.org/W4416713002","https://openalex.org/W4417077660","https://openalex.org/W4417081422","https://openalex.org/W4417156118","https://openalex.org/W4417298428","https://openalex.org/W4417511325","https://openalex.org/W4417530506","https://openalex.org/W6929499471","https://openalex.org/W7077044503","https://openalex.org/W7090943534","https://openalex.org/W7101563876","https://openalex.org/W7101885942","https://openalex.org/W7102546926","https://openalex.org/W7106804492","https://openalex.org/W7116855701","https://openalex.org/W7124738905","https://openalex.org/W7125282725","https://openalex.org/W7125896950","https://openalex.org/W7125985260","https://openalex.org/W7127380112","https://openalex.org/W7130402983","https://openalex.org/W7148703782","https://openalex.org/W7155005078","https://openalex.org/W7160626267","https://openalex.org/W7166846580"],"related_works":["https://openalex.org/W7165612465","https://openalex.org/W4415035193","https://openalex.org/W4416954965","https://openalex.org/W7160626267","https://openalex.org/W7155005078","https://openalex.org/W4406747045","https://openalex.org/W6929499471","https://openalex.org/W4407245356","https://openalex.org/W7221401940","https://openalex.org/W6948236204"],"abstract_inverted_index":{"While":[0],"adaptive":[1],"gradient":[2,64],"methods":[3,43],"are":[4,162,189],"the":[5,53,60,85,135,153,167],"workhorse":[6],"of":[7,37,62,87,105,139,156],"modern":[8],"machine":[9],"learning,":[10],"sign-based":[11,39,157],"optimization":[12,141],"algorithms":[13],"such":[14],"as":[15],"Lion":[16,108],"and":[17,57,107,127,182],"Muon":[18,126],"have":[19],"recently":[20],"demonstrated":[21],"superior":[22],"empirical":[23,154],"performance":[24],"over":[25],"AdamW":[26],"in":[27,70],"training":[28],"large":[29],"language":[30,71],"models":[31,188],"(LLM).":[32],"However,":[33],"a":[34,66,77,148],"theoretical":[35,150,180],"understanding":[36],"why":[38],"updates":[40],"outperform":[41],"variance-adapted":[42],"remains":[44],"elusive.":[45],"In":[46],"this":[47,97],"paper,":[48],"we":[49,75,100,121],"aim":[50],"to":[51,125,132,165],"bridge":[52],"gap":[54],"between":[55],"theory":[56],"practice":[58],"through":[59],"lens":[61],"heavy-tailed":[63,80,143],"noise,":[65],"phenomenon":[67],"frequently":[68],"observed":[69],"modeling":[72],"tasks.":[73],"Theoretically,":[74],"introduce":[76],"novel":[78],"generalized":[79,110],"noise":[81,98,187],"condition":[82],"that":[83,160,184],"captures":[84],"behavior":[86],"LLMs":[88],"more":[89],"accurately":[90],"than":[91],"standard":[92],"finite":[93],"variance":[94],"assumptions.":[95],"Under":[96],"model,":[99],"establish":[101],"sharp":[102],"convergence":[103],"rates":[104],"SignSGD":[106],"for":[109,152],"smooth":[111],"function":[112],"classes,":[113],"matching":[114],"or":[115],"surpassing":[116],"previous":[117],"best-known":[118],"bounds.":[119],"Furthermore,":[120],"extend":[122],"our":[123,133,179,185],"analysis":[124,138],"Muonlight,":[128],"providing":[129],"what":[130],"is,":[131],"knowledge,":[134],"first":[136],"rigorous":[137],"matrix":[140],"under":[142],"stochasticity.":[144],"These":[145],"results":[146],"offer":[147],"strong":[149],"justification":[151],"superiority":[155],"optimizers,":[158],"showcasing":[159],"they":[161],"naturally":[163],"suited":[164],"handle":[166],"noisy":[168],"gradients":[169],"associated":[170],"with":[171,191],"heavy":[172],"tails.":[173],"Empirically,":[174],"LLM":[175],"pretraining":[176],"experiments":[177],"validate":[178],"insights":[181],"confirm":[183],"proposed":[186],"well-aligned":[190],"practice.":[192]},"counts_by_year":[{"year":2026,"cited_by_count":15},{"year":2025,"cited_by_count":1}],"updated_date":"2026-11-12T12:28:57.059296","created_date":"2026-02-11T00:00:00"}