{"id":"https://openalex.org/W7161134047","doi":"https://doi.org/10.48550/arxiv.2605.12756","title":"Uncovering Symmetry Transfer in Large Language Models via Layer-Peeled Optimization","display_name":"Uncovering Symmetry Transfer in Large Language Models via Layer-Peeled Optimization","publication_year":2026,"publication_date":"2026-05-12","ids":{"openalex":"https://openalex.org/W7161134047","doi":"https://doi.org/10.48550/arxiv.2605.12756"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.12756","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12756","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.12756","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101264788","display_name":"Zhehang Du","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Du, Zhehang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136140870","display_name":"Hangfeng He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Hangfeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136109131","display_name":"Weijie Su","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Su, Weijie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.43059998750686646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.43059998750686646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.15129999816417694,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.034699998795986176,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5461000204086304},{"id":"https://openalex.org/keywords/homogeneous-space","display_name":"Homogeneous space","score":0.5008999705314636},{"id":"https://openalex.org/keywords/matrix","display_name":"Matrix (chemical analysis)","score":0.49630001187324524},{"id":"https://openalex.org/keywords/optimization-problem","display_name":"Optimization problem","score":0.47999998927116394},{"id":"https://openalex.org/keywords/permutation","display_name":"Permutation (music)","score":0.4309000074863434},{"id":"https://openalex.org/keywords/invariant","display_name":"Invariant (physics)","score":0.42089998722076416},{"id":"https://openalex.org/keywords/symmetry-group","display_name":"Symmetry group","score":0.42080000042915344},{"id":"https://openalex.org/keywords/convex-optimization","display_name":"Convex optimization","score":0.40639999508857727},{"id":"https://openalex.org/keywords/projection","display_name":"Projection (relational algebra)","score":0.3910999894142151}],"concepts":[{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5957000255584717},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5461000204086304},{"id":"https://openalex.org/C96469262","wikidata":"https://www.wikidata.org/wiki/Q1324364","display_name":"Homogeneous space","level":2,"score":0.5008999705314636},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.49630001187324524},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.47999998927116394},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4480000138282776},{"id":"https://openalex.org/C21308566","wikidata":"https://www.wikidata.org/wiki/Q7169365","display_name":"Permutation (music)","level":2,"score":0.4309000074863434},{"id":"https://openalex.org/C190470478","wikidata":"https://www.wikidata.org/wiki/Q2370229","display_name":"Invariant (physics)","level":2,"score":0.42089998722076416},{"id":"https://openalex.org/C44306375","wikidata":"https://www.wikidata.org/wiki/Q902019","display_name":"Symmetry group","level":2,"score":0.42080000042915344},{"id":"https://openalex.org/C157972887","wikidata":"https://www.wikidata.org/wiki/Q463359","display_name":"Convex optimization","level":3,"score":0.40639999508857727},{"id":"https://openalex.org/C57493831","wikidata":"https://www.wikidata.org/wiki/Q3134666","display_name":"Projection (relational algebra)","level":2,"score":0.3910999894142151},{"id":"https://openalex.org/C41045048","wikidata":"https://www.wikidata.org/wiki/Q202843","display_name":"Linear programming","level":2,"score":0.39089998602867126},{"id":"https://openalex.org/C62438384","wikidata":"https://www.wikidata.org/wiki/Q331350","display_name":"Simplex","level":2,"score":0.37860000133514404},{"id":"https://openalex.org/C84140500","wikidata":"https://www.wikidata.org/wiki/Q851512","display_name":"Permutation matrix","level":3,"score":0.3718999922275543},{"id":"https://openalex.org/C2779886137","wikidata":"https://www.wikidata.org/wiki/Q21030012","display_name":"Symmetry (geometry)","level":2,"score":0.3666999936103821},{"id":"https://openalex.org/C2778023678","wikidata":"https://www.wikidata.org/wiki/Q554403","display_name":"Duality (order theory)","level":2,"score":0.32679998874664307},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.326200008392334},{"id":"https://openalex.org/C115973184","wikidata":"https://www.wikidata.org/wiki/Q245457","display_name":"Circulant matrix","level":2,"score":0.3237000107765198},{"id":"https://openalex.org/C42355184","wikidata":"https://www.wikidata.org/wiki/Q1361088","display_name":"Matrix decomposition","level":3,"score":0.2944999933242798},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.29120001196861267},{"id":"https://openalex.org/C112680207","wikidata":"https://www.wikidata.org/wiki/Q714886","display_name":"Regular polygon","level":2,"score":0.2881999909877777},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.27790001034736633},{"id":"https://openalex.org/C101901036","wikidata":"https://www.wikidata.org/wiki/Q2269096","display_name":"Semidefinite programming","level":2,"score":0.2768999934196472},{"id":"https://openalex.org/C2781311116","wikidata":"https://www.wikidata.org/wiki/Q83306","display_name":"Group (periodic table)","level":2,"score":0.2578999996185303},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.2547000050544739},{"id":"https://openalex.org/C17349429","wikidata":"https://www.wikidata.org/wiki/Q1049914","display_name":"Matrix multiplication","level":3,"score":0.2538999915122986},{"id":"https://openalex.org/C144521790","wikidata":"https://www.wikidata.org/wiki/Q134164","display_name":"Simplex algorithm","level":3,"score":0.2531999945640564},{"id":"https://openalex.org/C11252640","wikidata":"https://www.wikidata.org/wiki/Q243723","display_name":"Monomial","level":2,"score":0.25279998779296875}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.12756","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12756","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.12756","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12756","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"are":[4,83],"pretrained":[5],"by":[6,39,55],"minimizing":[7],"the":[8,28,57,79,86,90,103,112,116,119,123,125,133,138,142,157,170,183,191,196,206,235],"cross-entropy":[9],"loss":[10],"for":[11,53,151,217,226],"next-token":[12,81],"prediction.":[13],"In":[14],"this":[15,20,37,71],"paper,":[16],"we":[17,99,167,239],"study":[18],"whether":[19],"optimization":[21,44,66,73],"strategy":[22],"can":[23],"induce":[24],"geometric":[25,262],"structure":[26],"in":[27,78,93,195],"learned":[29],"model":[30,92],"weights":[31],"and":[32,61,132,141,187,220],"context":[33,63,143,188],"embeddings.":[34],"We":[35],"approach":[36],"problem":[38,210],"analyzing":[40],"a":[41,49,94,107,177,222,231],"constrained":[42,207],"layer-peeled":[43,91],"program,":[45],"which":[46],"serves":[47],"as":[48,65,111,148],"mathematically":[50],"tractable":[51],"surrogate":[52],"LLMs":[54,244],"treating":[56],"output":[58,139,173],"projection":[59,174],"matrix":[60,128,175,186],"last-layer":[62],"embeddings":[64,144,189],"variables.":[67],"Our":[68],"analysis":[69],"of":[70,89,115,122,136,234],"nonconvex":[72,208],"program":[74],"demonstrates":[75],"that":[76,101,169,242],"symmetries":[77,193,247],"target":[80,104,153],"distributions":[82,154],"transferred":[84],"to":[85,204,211,221],"global":[87,171],"minimizers":[88],"precise":[95],"group-theoretic":[96],"sense.":[97],"Specifically,":[98],"prove":[100],"when":[102],"tokens":[105],"exhibit":[106,246],"cyclic-shift":[108],"symmetry":[109,219],"(such":[110],"seven":[113],"days":[114],"week":[117],"or":[118],"twelve":[120],"months":[121],"year),":[124],"optimal":[126,172,184,236],"logit":[127,185],"is":[129,203],"exactly":[130],"circulant,":[131],"Gram":[134],"matrices":[135],"both":[137],"projections":[140],"form":[145],"circulant":[146],"geometries":[147],"well.":[149],"Next,":[150],"exchangeable":[152],"invariant":[155],"under":[156,163],"symmetric":[158],"group":[159,165],"and,":[160],"more":[161],"generally,":[162],"two-transitive":[164],"actions,":[166],"show":[168],"forms":[176],"simplex":[178],"equiangular":[179],"tight":[180],"frame,":[181],"while":[182],"inherit":[190],"permutation":[192,227],"present":[194],"input":[197],"data.":[198],"A":[199],"key":[200],"technical":[201],"step":[202],"reduce":[205],"factorized":[209],"an":[212],"explicit":[213,258],"logit-level":[214],"convex":[215],"characterization":[216,233],"cyclic":[218],"symmetry-based":[223],"lower":[224],"bound":[225],"symmetry,":[228],"together":[229],"with":[230,249],"sharp":[232],"factorization.":[237],"Finally,":[238],"empirically":[240],"demonstrate":[241],"open-source":[243],"naturally":[245],"consistent":[248],"our":[250],"theoretical":[251],"predictions,":[252],"despite":[253],"being":[254],"trained":[255],"without":[256],"any":[257],"regularization":[259],"promoting":[260],"such":[261],"structure.":[263]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-15T00:00:00"}