{"id":"https://openalex.org/W7162656723","doi":"https://doi.org/10.48550/arxiv.2605.27733","title":"Can Entry-Wise Clipping Give Spectral Control of Stochastic Gradients?","display_name":"Can Entry-Wise Clipping Give Spectral Control of Stochastic Gradients?","publication_year":2026,"publication_date":"2026-05-26","ids":{"openalex":"https://openalex.org/W7162656723","doi":"https://doi.org/10.48550/arxiv.2605.27733"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.27733","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27733","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.27733","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125418595","display_name":"Zitao Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Zitao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002412726","display_name":"Site Bai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bai, Cedar Site","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137280614","display_name":"Zhe Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Zhe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048046167","display_name":"Brian Bullins","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bullins, Brian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137291007","display_name":"David F. Gleich","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gleich, David F.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.6437000036239624,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.6437000036239624,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10048","display_name":"Particle physics theoretical and experimental studies","score":0.03880000114440918,"subfield":{"id":"https://openalex.org/subfields/3106","display_name":"Nuclear and High Energy Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.03830000013113022,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.6417999863624573},{"id":"https://openalex.org/keywords/clipping","display_name":"Clipping (morphology)","score":0.6279000043869019},{"id":"https://openalex.org/keywords/normalization","display_name":"Normalization (sociology)","score":0.5400999784469604},{"id":"https://openalex.org/keywords/gaussian","display_name":"Gaussian","score":0.48590001463890076},{"id":"https://openalex.org/keywords/autocorrelation","display_name":"Autocorrelation","score":0.45649999380111694},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.45509999990463257},{"id":"https://openalex.org/keywords/gaussian-noise","display_name":"Gaussian noise","score":0.4124999940395355},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.4072999954223633},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.38909998536109924}],"concepts":[{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.6417999863624573},{"id":"https://openalex.org/C2776848632","wikidata":"https://www.wikidata.org/wiki/Q853463","display_name":"Clipping (morphology)","level":2,"score":0.6279000043869019},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5496000051498413},{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.5400999784469604},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5285000205039978},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.48590001463890076},{"id":"https://openalex.org/C5297727","wikidata":"https://www.wikidata.org/wiki/Q786970","display_name":"Autocorrelation","level":2,"score":0.45649999380111694},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.45509999990463257},{"id":"https://openalex.org/C4199805","wikidata":"https://www.wikidata.org/wiki/Q2725903","display_name":"Gaussian noise","level":2,"score":0.4124999940395355},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.4072999954223633},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.38909998536109924},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3871999979019165},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.384799987077713},{"id":"https://openalex.org/C177918212","wikidata":"https://www.wikidata.org/wiki/Q803623","display_name":"Perturbation (astronomy)","level":2,"score":0.35989999771118164},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.34060001373291016},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.3353999853134155},{"id":"https://openalex.org/C30049272","wikidata":"https://www.wikidata.org/wiki/Q6555326","display_name":"Spectral density estimation","level":3,"score":0.33340001106262207},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.33329999446868896},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.32100000977516174},{"id":"https://openalex.org/C168110828","wikidata":"https://www.wikidata.org/wiki/Q1331626","display_name":"Spectral density","level":2,"score":0.3172000050544739},{"id":"https://openalex.org/C29265498","wikidata":"https://www.wikidata.org/wiki/Q7047719","display_name":"Noise measurement","level":3,"score":0.31470000743865967},{"id":"https://openalex.org/C152822103","wikidata":"https://www.wikidata.org/wiki/Q7575207","display_name":"Spectral shape analysis","level":3,"score":0.2980000078678131},{"id":"https://openalex.org/C2983668108","wikidata":"https://www.wikidata.org/wiki/Q280453","display_name":"Spectral analysis","level":3,"score":0.2824999988079071},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.2793999910354614},{"id":"https://openalex.org/C104267543","wikidata":"https://www.wikidata.org/wiki/Q208163","display_name":"Signal processing","level":3,"score":0.2727999985218048},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.265500009059906},{"id":"https://openalex.org/C64812099","wikidata":"https://www.wikidata.org/wiki/Q176604","display_name":"Random matrix","level":3,"score":0.2630000114440918},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.2531999945640564}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.27733","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27733","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.27733","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27733","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Training":[0],"instabilities":[1],"such":[2,94],"as":[3],"loss":[4],"spikes":[5],"are":[6],"frequently":[7],"the":[8,26,45,113,151],"result":[9],"of":[10,15,48,93,143,164],"stochastic":[11],"gradient":[12,74],"noise.":[13,129],"Because":[14],"rare":[16],"expressions":[17],"in":[18],"language":[19],"training":[20,144],"data,":[21],"and":[22,31,84],"multiple":[23],"layer":[24],"composition,":[25],"noise":[27,75],"impact":[28],"is":[29],"heavy-tailed":[30,82],"survives":[32],"mini-batch":[33],"averaging.":[34],"Existing":[35],"remedies":[36],"trade":[37],"off":[38],"structure":[39,47],"against":[40],"cost:":[41],"vector-norm":[42],"clipping":[43,153],"ignores":[44],"matrix":[46],"weight":[49],"updates,":[50],"while":[51],"spectral":[52,103,155],"normalization":[53,156],"(e.g.,":[54],"Muon":[55],"(Jordan":[56],"et":[57],"al.,":[58],"2024))":[59],"respects":[60],"it":[61],"at":[62],"additional":[63],"cost.":[64],"We":[65,122,146],"show":[66],"that":[67,133,149],"this":[68],"trade-off":[69],"can":[70],"be":[71,78],"balanced.":[72],"Real":[73],"appears":[76],"to":[77,80],"similar":[79],"entry-wise":[81,100,115,152],"contamination,":[83],"a":[85,90,98,109,118,158],"first-order":[86],"perturbation":[87],"analysis":[88],"reveals":[89],"localization":[91],"property":[92],"noise,":[95],"under":[96,117,127],"which":[97],"simple":[99],"method":[101],"achieves":[102],"control.":[104],"Exploiting":[105],"this,":[106],"we":[107,131],"derive":[108],"tractable":[110],"surrogate":[111],"for":[112],"Bayes-optimal":[114],"estimator":[116],"Gaussian":[119],"signal":[120],"prior.":[121],"establish":[123],"$O(\u03b5^{-4})$":[124],"convergence":[125],"guarantee":[126],"Cauchy-contaminated":[128],"Empirically,":[130],"find":[132,148],"smooth":[134],"shrinkage":[135],"improves":[136],"Adam":[137],"on":[138,162],"NanoGPT":[139],"pretraining,":[140],"saving":[141,161],"${\\sim}7\\%$":[142],"tokens.":[145],"further":[147],"applying":[150],"before":[154],"yields":[157],"${\\sim}2\\%$":[159],"token":[160],"top":[163],"Muon.":[165]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-29T00:00:00"}