{"id":"https://openalex.org/W7167712301","doi":"https://doi.org/10.48550/arxiv.2607.05872","title":"No Subspace to Track: Non-Identifiability and Optimizer State in Low-Rank Training","display_name":"No Subspace to Track: Non-Identifiability and Optimizer State in Low-Rank Training","publication_year":2026,"publication_date":"2026-07-07","ids":{"openalex":"https://openalex.org/W7167712301","doi":"https://doi.org/10.48550/arxiv.2607.05872"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.05872","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.05872","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.05872","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140288865","display_name":"Noel Thomas","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Thomas, Noel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5140288865"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.3756999969482422,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.3756999969482422,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11094","display_name":"Face Recognition and Perception","score":0.049300000071525574,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.04820000007748604,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/subspace-topology","display_name":"Subspace topology","score":0.833899974822998},{"id":"https://openalex.org/keywords/disjoint-sets","display_name":"Disjoint sets","score":0.6360999941825867},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.59579998254776},{"id":"https://openalex.org/keywords/interchangeability","display_name":"Interchangeability","score":0.5893999934196472},{"id":"https://openalex.org/keywords/perplexity","display_name":"Perplexity","score":0.5547000169754028},{"id":"https://openalex.org/keywords/moment","display_name":"Moment (physics)","score":0.5404999852180481},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.477400004863739},{"id":"https://openalex.org/keywords/object","display_name":"Object (grammar)","score":0.43459999561309814},{"id":"https://openalex.org/keywords/supervisor","display_name":"Supervisor","score":0.4171000123023987}],"concepts":[{"id":"https://openalex.org/C32834561","wikidata":"https://www.wikidata.org/wiki/Q660730","display_name":"Subspace topology","level":2,"score":0.833899974822998},{"id":"https://openalex.org/C45340560","wikidata":"https://www.wikidata.org/wiki/Q215382","display_name":"Disjoint sets","level":2,"score":0.6360999941825867},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.59579998254776},{"id":"https://openalex.org/C2779606619","wikidata":"https://www.wikidata.org/wiki/Q17092524","display_name":"Interchangeability","level":2,"score":0.5893999934196472},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5547999739646912},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.5547000169754028},{"id":"https://openalex.org/C179254644","wikidata":"https://www.wikidata.org/wiki/Q13222844","display_name":"Moment (physics)","level":2,"score":0.5404999852180481},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5133000016212463},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.477400004863739},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4374000132083893},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.43459999561309814},{"id":"https://openalex.org/C2779110517","wikidata":"https://www.wikidata.org/wiki/Q1240788","display_name":"Supervisor","level":2,"score":0.4171000123023987},{"id":"https://openalex.org/C74050887","wikidata":"https://www.wikidata.org/wiki/Q848368","display_name":"Rotation (mathematics)","level":2,"score":0.3677000105381012},{"id":"https://openalex.org/C184050105","wikidata":"https://www.wikidata.org/wiki/Q273163","display_name":"Isotropy","level":2,"score":0.3562999963760376},{"id":"https://openalex.org/C2781315470","wikidata":"https://www.wikidata.org/wiki/Q193623","display_name":"Binomial (polynomial)","level":2,"score":0.3562999963760376},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.3458999991416931},{"id":"https://openalex.org/C529865628","wikidata":"https://www.wikidata.org/wiki/Q1790740","display_name":"Manifold (fluid mechanics)","level":2,"score":0.34549999237060547},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3249000012874603},{"id":"https://openalex.org/C2776865275","wikidata":"https://www.wikidata.org/wiki/Q311666","display_name":"Projector","level":2,"score":0.30090001225471497},{"id":"https://openalex.org/C191287063","wikidata":"https://www.wikidata.org/wiki/Q543281","display_name":"Glitch","level":3,"score":0.2872999906539917},{"id":"https://openalex.org/C62611344","wikidata":"https://www.wikidata.org/wiki/Q1062658","display_name":"Node (physics)","level":2,"score":0.2838999927043915},{"id":"https://openalex.org/C137800194","wikidata":"https://www.wikidata.org/wiki/Q11713455","display_name":"Interpolation (computer graphics)","level":3,"score":0.2797999978065491},{"id":"https://openalex.org/C12362212","wikidata":"https://www.wikidata.org/wiki/Q728435","display_name":"Linear subspace","level":2,"score":0.27810001373291016},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.2761000096797943},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2703999876976013},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.2646999955177307},{"id":"https://openalex.org/C148043351","wikidata":"https://www.wikidata.org/wiki/Q4456944","display_name":"Current (fluid)","level":2,"score":0.26159998774528503},{"id":"https://openalex.org/C184720557","wikidata":"https://www.wikidata.org/wiki/Q7825049","display_name":"Topology (electrical circuits)","level":2,"score":0.25929999351501465},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.2581999897956848}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.05872","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.05872","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.05872","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.05872","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Memory-efficient":[0],"optimizers":[1],"such":[2,43],"as":[3,60,62,140,166],"GaLore":[4,226,246,264],"train":[5],"large":[6],"language":[7],"models":[8],"by":[9,89],"projecting":[10],"gradients":[11,204],"onto":[12],"a":[13,24,36,114,167,252,275],"rank-r":[14],"subspace":[15,22,50,155,260],"recomputed":[16],"every":[17],"T":[18,65],"steps,":[19],"assuming":[20],"this":[21],"is":[23,41,87,179,249],"slowly":[25],"drifting":[26],"object":[27],"that":[28,34],"can":[29],"be":[30],"tracked.":[31],"We":[32],"show":[33],"beyond":[35],"small":[37,250],"reproducible":[38,123,279],"core,":[39],"there":[40],"no":[42,150],"object.":[44],"Two":[45],"estimates":[46,63],"of":[47,71,119,146,169],"the":[48,53,72,81,130,135,144,154,175,185,190,196,198,206,234,240,247,278],"top-r":[49],"computed":[51,64],"at":[52,77,84,222],"same":[54],"step":[55],"from":[56,102,162],"disjoint":[57],"minibatches":[58],"disagree":[59],"much":[61],"steps":[66,214],"apart":[67],"(0.73":[68],"vs":[69],"0.74":[70],"maximal":[73],"chordal":[74],"distance":[75],"sqrt(2r),":[76],"Pythia-160M":[78],"with":[79,108],"r=128):":[80],"apparent":[82],"rotation":[83],"each":[85,164],"refresh":[86,165],"dominated":[88],"estimator":[90],"noise.":[91],"This":[92],"holds":[93],"across":[94,124],"four":[95],"model":[96],"families":[97],"in":[98,113],"three":[99],"architecture":[100],"classes":[101],"70M":[103],"to":[104,237,271],"6.9B":[105],"parameters,":[106],"strengthening":[107],"scale,":[109],"and":[110,126,205,251,269],"more":[111],"weakly":[112],"vision":[115],"transformer.":[116],"Only":[117],"~39":[118],"128":[120],"directions":[121],"are":[122],"minibatches,":[125],"averaging":[127,134,151],"cannot":[128],"recover":[129],"rest:":[131],"under":[132,202],"N-fold":[133],"gradient's":[136],"spectral":[137],"tail":[138],"shrinks":[139],"N^(-1/4)":[141],"rather":[142],"than":[143,184],"N^(-1/2)":[145],"pure":[147],"noise,":[148],"so":[149],"budget":[152],"makes":[153],"well":[156],"defined.":[157],"What":[158],"helps":[159,239],"instead":[160],"follows":[161],"treating":[163],"change":[168],"coordinates":[170],"for":[171,244],"Adam's":[172],"state.":[173],"Carrying":[174],"second":[176],"moment":[177,192],"blindly":[178],"provably":[180],"about":[181],"(r-k*)/2":[182],"worse":[183],"best":[186,229],"rotation-blind":[187],"estimator,":[188],"while":[189],"first":[191],"transports":[193],"exactly":[194],"through":[195],"rotation,":[197],"optimal":[199],"linear":[200],"map":[201],"isotropic":[203],"rule":[207],"LDAdam":[208,218],"uses.":[209],"At":[210],"1B":[211],"over":[212],"40k":[213],"(3":[215],"seeds),":[216],"full":[217],"reaches":[219],"18.7":[220],"perplexity":[221],"beta2=0.999,":[223],"beating":[224],"untransported":[225],"after":[227],"its":[228],"beta2":[230],"fix":[231],"(19.3);":[232],"shortening":[233],"second-moment":[235],"memory":[236],"beta2=0.99":[238],"refreshing":[241],"optimizers,":[242],"though":[243],"canonical":[245],"effect":[248],"full-rank":[253],"control":[254],"reverses":[255],"it.":[256],"One":[257],"measurable":[258],"fact,":[259],"non-identifiability,":[261],"clarifies":[262],"why":[263],"works,":[265],"which":[266],"patches":[267],"work,":[268],"what":[270],"check":[272],"before":[273],"trusting":[274],"low-rank":[276],"assumption:":[277],"rank":[280],"k*.":[281]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-09T00:00:00"}