{"id":"https://openalex.org/W7213930786","doi":"https://doi.org/10.48550/arxiv.2609.21039","title":"Stiefel-AdamW: Geometry-Aware AdamW for Linear Factorization Blocks","display_name":"Stiefel-AdamW: Geometry-Aware AdamW for Linear Factorization Blocks","publication_year":2026,"publication_date":"2026-09-17","ids":{"openalex":"https://openalex.org/W7213930786","doi":"https://doi.org/10.48550/arxiv.2609.21039"},"language":null,"primary_location":{"id":"pmh:oai:arXiv.org:2609.21039","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.21039","pdf_url":"https://arxiv.org/pdf/2609.21039","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.21039","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072627792","display_name":"Emanuele Zangrando","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210150763","display_name":"Gran Sasso Science Institute","ror":"https://ror.org/043qcb444","country_code":"IT","type":"education","lineage":["https://openalex.org/I160013858","https://openalex.org/I4210150763"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Zangrando, Emanuele","raw_affiliation_strings":["Gran Sasso Science Institute , L'Aquila , Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Gran Sasso Science Institute , L'Aquila , Italy","institution_ids":["https://openalex.org/I4210150763"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031520014","display_name":"Marco Sutti","orcid":"https://orcid.org/0000-0002-8410-1372","observed_orcids":["https://orcid.org/0000-0002-8410-1372"]},"institutions":[{"id":"https://openalex.org/I4210150763","display_name":"Gran Sasso Science Institute","ror":"https://ror.org/043qcb444","country_code":"IT","type":"education","lineage":["https://openalex.org/I160013858","https://openalex.org/I4210150763"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Sutti, Marco","raw_affiliation_strings":["Gran Sasso Science Institute , L'Aquila , Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Gran Sasso Science Institute , L'Aquila , Italy","institution_ids":["https://openalex.org/I4210150763"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043752696","display_name":"Francesco Tudisco","orcid":"https://orcid.org/0000-0002-8150-4475","observed_orcids":["https://orcid.org/0000-0002-8150-4475"]},"institutions":[{"id":"https://openalex.org/I4210150763","display_name":"Gran Sasso Science Institute","ror":"https://ror.org/043qcb444","country_code":"IT","type":"education","lineage":["https://openalex.org/I160013858","https://openalex.org/I4210150763"]},{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB","IT"],"is_corresponding":false,"raw_author_name":"Tudisco, Francesco","raw_affiliation_strings":["Gran Sasso Science Institute , L'Aquila , Italy","University of Edinburgh , Edinburgh , UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Gran Sasso Science Institute , L'Aquila , Italy","institution_ids":["https://openalex.org/I4210150763"]},{"raw_affiliation_string":"University of Edinburgh , Edinburgh , UK","institution_ids":["https://openalex.org/I98677209"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0027000000700354576,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0010000000474974513,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/stiefel-manifold","display_name":"Stiefel manifold","score":0.921999990940094},{"id":"https://openalex.org/keywords/adamw","display_name":"AdamW","score":0.6069999933242798},{"id":"https://openalex.org/keywords/riemannian-optimization","display_name":"Riemannian optimization","score":0.8339999914169312},{"id":"https://openalex.org/keywords/non-unique-factorization","display_name":"non-unique factorization","score":0.6940000057220459},{"id":"https://openalex.org/keywords/low-rank-adapters","display_name":"low-rank adapters","score":0.5220000147819519},{"id":"https://openalex.org/keywords/lora","display_name":"LoRa","score":0.8370000123977661},{"id":"https://openalex.org/keywords/orthogonal-symmetry","display_name":"orthogonal symmetry","score":0.5619999766349792},{"id":"https://openalex.org/keywords/diagonal-preconditioning","display_name":"diagonal preconditioning","score":0.7099999785423279},{"id":"https://openalex.org/keywords/method-of-moments","display_name":"method of moments","score":0.5839999914169312},{"id":"https://openalex.org/keywords/tangent-space-projection","display_name":"tangent space projection","score":0.7429999709129333}],"concepts":[{"id":"https://openalex.org/C187834632","wikidata":"https://www.wikidata.org/wiki/Q188804","display_name":"Factorization","level":2,"score":0.6753000020980835},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5536999702453613},{"id":"https://openalex.org/C57493831","wikidata":"https://www.wikidata.org/wiki/Q3134666","display_name":"Projection (relational algebra)","level":2,"score":0.4952999949455261},{"id":"https://openalex.org/C529865628","wikidata":"https://www.wikidata.org/wiki/Q1790740","display_name":"Manifold (fluid mechanics)","level":2,"score":0.4767000079154968},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.44839999079704285},{"id":"https://openalex.org/C129782007","wikidata":"https://www.wikidata.org/wiki/Q162886","display_name":"Euclidean geometry","level":2,"score":0.4404999911785126},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.40610000491142273},{"id":"https://openalex.org/C130367717","wikidata":"https://www.wikidata.org/wiki/Q189791","display_name":"Diagonal","level":2,"score":0.40230000019073486},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.3970000147819519},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.3921999931335449},{"id":"https://openalex.org/C42355184","wikidata":"https://www.wikidata.org/wiki/Q1361088","display_name":"Matrix decomposition","level":3,"score":0.3644999861717224},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.36250001192092896},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.3537999987602234},{"id":"https://openalex.org/C136119220","wikidata":"https://www.wikidata.org/wiki/Q1000660","display_name":"Algebra over a field","level":2,"score":0.3467999994754791},{"id":"https://openalex.org/C120174047","wikidata":"https://www.wikidata.org/wiki/Q847073","display_name":"Euclidean distance","level":2,"score":0.3449999988079071},{"id":"https://openalex.org/C612670","wikidata":"https://www.wikidata.org/wiki/Q7616373","display_name":"Stiefel manifold","level":2,"score":0.3345000147819519},{"id":"https://openalex.org/C186450821","wikidata":"https://www.wikidata.org/wiki/Q17295","display_name":"Euclidean space","level":2,"score":0.334199994802475},{"id":"https://openalex.org/C134978465","wikidata":"https://www.wikidata.org/wiki/Q1654069","display_name":"Incomplete LU factorization","level":4,"score":0.3325999975204468},{"id":"https://openalex.org/C192939610","wikidata":"https://www.wikidata.org/wiki/Q188444","display_name":"Differential geometry","level":2,"score":0.3133000135421753},{"id":"https://openalex.org/C184720557","wikidata":"https://www.wikidata.org/wiki/Q7825049","display_name":"Topology (electrical circuits)","level":2,"score":0.3118000030517578},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.3100999891757965},{"id":"https://openalex.org/C6802819","wikidata":"https://www.wikidata.org/wiki/Q1072174","display_name":"Linear system","level":2,"score":0.30000001192092896},{"id":"https://openalex.org/C179254644","wikidata":"https://www.wikidata.org/wiki/Q13222844","display_name":"Moment (physics)","level":2,"score":0.29190000891685486},{"id":"https://openalex.org/C2780615836","wikidata":"https://www.wikidata.org/wiki/Q2471869","display_name":"USable","level":2,"score":0.2906000018119812},{"id":"https://openalex.org/C2779886137","wikidata":"https://www.wikidata.org/wiki/Q21030012","display_name":"Symmetry (geometry)","level":2,"score":0.2822999954223633},{"id":"https://openalex.org/C41045048","wikidata":"https://www.wikidata.org/wiki/Q202843","display_name":"Linear programming","level":2,"score":0.2782999873161316},{"id":"https://openalex.org/C32834561","wikidata":"https://www.wikidata.org/wiki/Q660730","display_name":"Subspace topology","level":2,"score":0.27129998803138733},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.27059999108314514},{"id":"https://openalex.org/C190502265","wikidata":"https://www.wikidata.org/wiki/Q17069496","display_name":"MNIST database","level":3,"score":0.25679999589920044},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2549000084400177}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.21039","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.21039","pdf_url":"https://arxiv.org/pdf/2609.21039","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.21039","doi":"https://doi.org/10.48550/arxiv.2609.21039","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.21039","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.21039","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.21039","pdf_url":"https://arxiv.org/pdf/2609.21039","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7213930786.pdf","grobid_xml":"https://content.openalex.org/works/W7213930786.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"A":[0],"pervasive":[1],"structural":[2],"pattern":[3],"in":[4,21,36,140],"modern":[5],"deep":[6],"learning":[7,62],"is":[8,53,138,162],"the":[9,16,51,77,100,105,110,126,141,168,173],"linear":[10],"factorization":[11,52,66],"block:":[12],"a":[13,48,83,116,150,154],"submodule":[14],"of":[15,44,176,189,199],"form":[17],"$W":[18],"=":[19],"BA$":[20],"which":[22,55],"two":[23],"parameter":[24],"matrices":[25],"are":[26,68],"multiplied":[27],"directly,":[28],"with":[29,71,145],"no":[30,211],"intervening":[31],"nonlinearity.":[32],"Such":[33],"blocks":[34,67,93],"appear":[35],"LoRA":[37],"adapters,":[38],"low-rank":[39],"compressed":[40],"layers,":[41],"query-key":[42],"products":[43],"self-attention,":[45],"and":[46,59,153,164,179,192,195],"share":[47],"common":[49],"pathology:":[50],"non-unique,":[54],"can":[56],"destabilize":[57],"training":[58],"limit":[60],"usable":[61],"rates.":[63],"Despite":[64],"this,":[65],"typically":[69],"optimized":[70],"standard":[72,180],"Euclidean":[73,143],"methods":[74,178],"that":[75,130,167],"ignore":[76],"underlying":[78],"geometry.":[79],"We":[80,183],"introduce":[81],"Stiefel-AdamW,":[82],"near":[84],"drop-in":[85],"replacement":[86],"for":[87,89],"AdamW":[88,132,161],"use":[90],"wherever":[91],"such":[92],"appear.":[94],"By":[95],"constraining":[96],"one":[97],"factor":[98,122],"on":[99,186,196,201],"Stiefel":[101],"manifold":[102,155],"while":[103,124],"leaving":[104],"other":[106],"Euclidean,":[107],"Stiefel-AdamW":[108,185],"relaxes":[109],"full":[111,197],"$\\mathrm{GL}(\\mathbb{R}^r)$":[112],"gauge":[113],"symmetry":[114],"to":[115],"compact":[117],"orthogonal":[118],"symmetry,":[119],"ruling":[120],"out":[121],"blow-up":[123],"retaining":[125],"coordinate-wise":[127],"diagonal":[128],"preconditioning":[129],"gives":[131],"its":[133],"practical":[134],"strength.":[135],"Moment":[136],"estimation":[137],"performed":[139],"ambient":[142],"space,":[144],"geometry":[146],"entering":[147],"only":[148],"through":[149],"tangent-space":[151],"projection":[152],"retraction.":[156],"The":[157],"implementation":[158],"overhead":[159],"over":[160,206,214],"minimal,":[163],"we":[165],"show":[166],"resulting":[169],"optimizer":[170],"inherits":[171],"both":[172],"stability":[174],"benefits":[175],"Riemannian":[177],"convergence":[181],"guarantees.":[182],"validate":[184],"LoRA-style":[187],"fine-tuning":[188],"GPT2,":[190],"ViT,":[191],"Mistral":[193],"7B":[194],"pretraining":[198],"GPT2":[200],"OpenWebText,":[202],"showing":[203],"consistent":[204],"improvements":[205],"strong":[207],"baselines":[208],"at":[209],"essentially":[210],"additional":[212],"cost":[213],"AdamW.":[215]},"counts_by_year":[],"updated_date":"2026-09-30T09:27:27.532063","created_date":"2026-09-22T00:00:00"}