{"id":"https://openalex.org/W7162569926","doi":"https://doi.org/10.48550/arxiv.2605.26489","title":"The Stability of Singular Distribution: A Spectral Perspective on the Two-Phase Dynamics of Language Model Pre-training","display_name":"The Stability of Singular Distribution: A Spectral Perspective on the Two-Phase Dynamics of Language Model Pre-training","publication_year":2026,"publication_date":"2026-05-26","ids":{"openalex":"https://openalex.org/W7162569926","doi":"https://doi.org/10.48550/arxiv.2605.26489"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.26489","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.26489","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.26489","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137118639","display_name":"Hongtao Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Hongtao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101918392","display_name":"Wenjie Zhou","orcid":"https://orcid.org/0000-0002-8385-7194"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Wenjie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137087589","display_name":"Chenxi Jia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jia, Chenxi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137133228","display_name":"Wei Chen","orcid":"https://orcid.org/0000-0002-3277-7629"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137162336","display_name":"Xueqi Cheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Xueqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.14270000159740448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.14270000159740448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.07090000063180923,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.04960000142455101,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.5805000066757202},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.5059000253677368},{"id":"https://openalex.org/keywords/singular-value","display_name":"Singular value","score":0.4571000039577484},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.4512999951839447},{"id":"https://openalex.org/keywords/dynamics","display_name":"Dynamics (music)","score":0.44179999828338623},{"id":"https://openalex.org/keywords/bounded-function","display_name":"Bounded function","score":0.4316999912261963},{"id":"https://openalex.org/keywords/trigonometric-functions","display_name":"Trigonometric functions","score":0.4018000066280365},{"id":"https://openalex.org/keywords/spectrum","display_name":"Spectrum (functional analysis)","score":0.3756999969482422}],"concepts":[{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.5805000066757202},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5613999962806702},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.5059000253677368},{"id":"https://openalex.org/C109282560","wikidata":"https://www.wikidata.org/wiki/Q4166054","display_name":"Singular value","level":3,"score":0.4571000039577484},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.4512999951839447},{"id":"https://openalex.org/C145912823","wikidata":"https://www.wikidata.org/wiki/Q113558","display_name":"Dynamics (music)","level":2,"score":0.44179999828338623},{"id":"https://openalex.org/C34388435","wikidata":"https://www.wikidata.org/wiki/Q2267362","display_name":"Bounded function","level":2,"score":0.4316999912261963},{"id":"https://openalex.org/C178009071","wikidata":"https://www.wikidata.org/wiki/Q93344","display_name":"Trigonometric functions","level":2,"score":0.4018000066280365},{"id":"https://openalex.org/C156778621","wikidata":"https://www.wikidata.org/wiki/Q1365748","display_name":"Spectrum (functional analysis)","level":2,"score":0.3756999969482422},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.3686000108718872},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.3594000041484833},{"id":"https://openalex.org/C22789450","wikidata":"https://www.wikidata.org/wiki/Q420904","display_name":"Singular value decomposition","level":2,"score":0.3490000069141388},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3010999858379364},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.2971000075340271},{"id":"https://openalex.org/C110121322","wikidata":"https://www.wikidata.org/wiki/Q865811","display_name":"Distribution (mathematics)","level":2,"score":0.2906999886035919},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.28600001335144043},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.2847000062465668},{"id":"https://openalex.org/C136272165","wikidata":"https://www.wikidata.org/wiki/Q4048889","display_name":"Singular spectrum analysis","level":3,"score":0.2669000029563904},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.2660999894142151},{"id":"https://openalex.org/C2781345722","wikidata":"https://www.wikidata.org/wiki/Q5308388","display_name":"Drop (telecommunication)","level":2,"score":0.262800008058548},{"id":"https://openalex.org/C2983668108","wikidata":"https://www.wikidata.org/wiki/Q280453","display_name":"Spectral analysis","level":3,"score":0.26089999079704285},{"id":"https://openalex.org/C2780388253","wikidata":"https://www.wikidata.org/wiki/Q5421508","display_name":"Exponent","level":2,"score":0.2565999925136566}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.26489","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.26489","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.26489","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.26489","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.7652955651283264,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"model":[2],"pre-training":[3,136],"typically":[4],"exhibits":[5],"a":[6,9,16,81,130],"two-phase":[7],"trajectory:":[8],"fast":[10],"initial":[11],"loss":[12,101],"drop":[13],"followed":[14],"by":[15,106],"prolonged":[17],"slow":[18],"improvement.":[19],"We":[20,46,113],"identify":[21],"an":[22,92],"underlying":[23],"spectral":[24,131],"phenomenon,":[25],"Stability":[26],"of":[27,100],"Singular":[28],"Distribution":[29],"(SoSD),":[30],"where":[31],"the":[32,53,98,107,110,126],"trace-normalized":[33],"singular":[34,111],"value":[35],"spectrum":[36],"stabilizes":[37],"early,":[38],"even":[39],"as":[40],"parameter":[41],"matrices":[42],"continue":[43],"to":[44,124],"evolve.":[45],"demonstrate":[47],"that":[48,86],"synchronization":[49],"between":[50],"SoSD":[51,94,127],"and":[52,64,75,119],"slow-descent":[54],"regime":[55],"is":[56],"widely":[57],"observed":[58],"across":[59],"diverse":[60],"architectures":[61],"(GPT-2,":[62],"LLaMA)":[63],"settings,":[65],"including":[66],"various":[67],"schedules":[68],"(Step-wise,":[69],"WSD,":[70],"Cosine":[71],"Decay),":[72],"weight":[73,88],"decays,":[74],"optimizers":[76],"(AdamW,":[77],"Muon).":[78],"By":[79],"analyzing":[80],"simplified":[82],"Transformer,":[83],"we":[84],"prove":[85],"growing":[87],"norms":[89],"inevitably":[90],"precipitate":[91],"early":[93],"threshold,":[95],"after":[96],"which":[97],"rate":[99],"decrease":[102],"becomes":[103],"theoretically":[104],"bounded":[105],"variation":[108],"in":[109],"distribution.":[112],"further":[114],"interpret":[115],"strategies":[116],"like":[117],"WSD":[118],"Muon":[120],"through":[121],"their":[122],"ability":[123],"modulate":[125],"scale,":[128],"offering":[129],"lens":[132],"for":[133],"understanding":[134],"efficient":[135],"dynamics.":[137]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-28T00:00:00"}