{"id":"https://openalex.org/W7161278272","doi":"https://doi.org/10.48550/arxiv.2605.14258","title":"Dynamics of the Transformer Residual Stream: Coupling Spectral Geometry to Network Topology","display_name":"Dynamics of the Transformer Residual Stream: Coupling Spectral Geometry to Network Topology","publication_year":2026,"publication_date":"2026-05-14","ids":{"openalex":"https://openalex.org/W7161278272","doi":"https://doi.org/10.48550/arxiv.2605.14258"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.14258","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14258","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.14258","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089995448","display_name":"Jesseba Fernando","orcid":"https://orcid.org/0000-0001-6776-947X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fernando, Jesseba","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5089601338","display_name":"Grigori Guitchounts","orcid":"https://orcid.org/0000-0003-0142-0149"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guitchounts, Grigori","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.42160001397132874,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.42160001397132874,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.21209999918937683,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.05950000137090683,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/jacobian-matrix-and-determinant","display_name":"Jacobian matrix and determinant","score":0.7312999963760376},{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.6453999876976013},{"id":"https://openalex.org/keywords/nonlinear-system","display_name":"Nonlinear system","score":0.5830000042915344},{"id":"https://openalex.org/keywords/perturbation","display_name":"Perturbation (astronomy)","score":0.557699978351593},{"id":"https://openalex.org/keywords/topology","display_name":"Topology (electrical circuits)","score":0.5404999852180481},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.5378000140190125},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.5065000057220459},{"id":"https://openalex.org/keywords/spectral-shape-analysis","display_name":"Spectral shape analysis","score":0.43970000743865967},{"id":"https://openalex.org/keywords/scalar","display_name":"Scalar (mathematics)","score":0.41690000891685486},{"id":"https://openalex.org/keywords/monotonic-function","display_name":"Monotonic function","score":0.3711000084877014}],"concepts":[{"id":"https://openalex.org/C200331156","wikidata":"https://www.wikidata.org/wiki/Q506041","display_name":"Jacobian matrix and determinant","level":2,"score":0.7312999963760376},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.6453999876976013},{"id":"https://openalex.org/C158622935","wikidata":"https://www.wikidata.org/wiki/Q660848","display_name":"Nonlinear system","level":2,"score":0.5830000042915344},{"id":"https://openalex.org/C177918212","wikidata":"https://www.wikidata.org/wiki/Q803623","display_name":"Perturbation (astronomy)","level":2,"score":0.557699978351593},{"id":"https://openalex.org/C184720557","wikidata":"https://www.wikidata.org/wiki/Q7825049","display_name":"Topology (electrical circuits)","level":2,"score":0.5404999852180481},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.5378000140190125},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5065000057220459},{"id":"https://openalex.org/C152822103","wikidata":"https://www.wikidata.org/wiki/Q7575207","display_name":"Spectral shape analysis","level":3,"score":0.43970000743865967},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.4345000088214874},{"id":"https://openalex.org/C57691317","wikidata":"https://www.wikidata.org/wiki/Q1289248","display_name":"Scalar (mathematics)","level":2,"score":0.41690000891685486},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.3831000030040741},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.37450000643730164},{"id":"https://openalex.org/C72169020","wikidata":"https://www.wikidata.org/wiki/Q194404","display_name":"Monotonic function","level":2,"score":0.3711000084877014},{"id":"https://openalex.org/C131584629","wikidata":"https://www.wikidata.org/wiki/Q4308705","display_name":"Coupling (piping)","level":2,"score":0.3504999876022339},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3352999985218048},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.3325999975204468},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.32829999923706055},{"id":"https://openalex.org/C158693339","wikidata":"https://www.wikidata.org/wiki/Q190524","display_name":"Eigenvalues and eigenvectors","level":2,"score":0.3269999921321869},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.32409998774528503},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.32409998774528503},{"id":"https://openalex.org/C38976095","wikidata":"https://www.wikidata.org/wiki/Q752641","display_name":"Asymmetry","level":2,"score":0.31130000948905945},{"id":"https://openalex.org/C199845137","wikidata":"https://www.wikidata.org/wiki/Q145490","display_name":"Network topology","level":2,"score":0.3061999976634979},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.30379998683929443},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.2992999851703644},{"id":"https://openalex.org/C17020691","wikidata":"https://www.wikidata.org/wiki/Q139677","display_name":"Operator (biology)","level":5,"score":0.2985000014305115},{"id":"https://openalex.org/C62799726","wikidata":"https://www.wikidata.org/wiki/Q190056","display_name":"Hilbert space","level":2,"score":0.2930999994277954},{"id":"https://openalex.org/C192939610","wikidata":"https://www.wikidata.org/wiki/Q188444","display_name":"Differential geometry","level":2,"score":0.2928999960422516},{"id":"https://openalex.org/C60292330","wikidata":"https://www.wikidata.org/wiki/Q1014065","display_name":"Hadamard transform","level":2,"score":0.29269999265670776},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.28929999470710754},{"id":"https://openalex.org/C176641082","wikidata":"https://www.wikidata.org/wiki/Q2446767","display_name":"Spectral signature","level":2,"score":0.28619998693466187},{"id":"https://openalex.org/C140820882","wikidata":"https://www.wikidata.org/wiki/Q732722","display_name":"Vortex","level":2,"score":0.2827000021934509},{"id":"https://openalex.org/C140532419","wikidata":"https://www.wikidata.org/wiki/Q249748","display_name":"Spectral radius","level":3,"score":0.28060001134872437},{"id":"https://openalex.org/C167879884","wikidata":"https://www.wikidata.org/wiki/Q727568","display_name":"Balanced flow","level":2,"score":0.26840001344680786},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.2653999924659729},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.26269999146461487},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.2612999975681305}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.14258","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14258","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.14258","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14258","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.41334268450737,"display_name":"Sustainable cities and communities","id":"https://metadata.un.org/sdg/11"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"are":[3,124],"remarkably":[4],"capable,":[5],"yet":[6],"how":[7],"computation":[8],"propagates":[9],"through":[10,82],"their":[11],"layers":[12,89,93],"remains":[13],"poorly":[14],"understood.":[15],"A":[16],"growing":[17],"line":[18],"of":[19,60,108,145,159],"work":[20],"treats":[21],"depth":[22,83],"as":[23,30],"discrete":[24],"time":[25],"and":[26,73,120,129,186],"the":[27,56,109,121,142,150,157,160,164,189],"residual":[28,110],"stream":[29],"a":[31,40,78,97,105,168,176],"dynamical":[32],"system,":[33],"where":[34],"each":[35],"layer's":[36],"nonlinear":[37],"update":[38],"has":[39],"local":[41,165],"linear":[42],"description.":[43],"However,":[44],"previous":[45],"analyses":[46],"have":[47],"relied":[48],"on":[49],"scalar":[50],"summaries":[51],"or":[52,153],"approximate":[53],"linearizations,":[54],"leaving":[55],"full":[57,66],"spectral":[58,80,178],"geometry":[59,179],"trained":[61],"LLMs":[62,72,181],"unknown.":[63],"We":[64,138],"perform":[65],"Jacobian":[67,151],"eigendecomposition":[68],"across":[69],"three":[70],"production--scale":[71],"show":[74,140],"that":[75,101,117,141,182],"training":[76],"installs":[77],"monotonic":[79],"gradient":[81,119],"--":[84,94],"from":[85],"non-normal,":[86],"rotation-dominated":[87],"early":[88],"to":[90,188],"near--symmetric":[91],"late":[92],"together":[95],"with":[96,156],"cumulative":[98],"low-rank":[99],"bottleneck":[100],"funnels":[102],"perturbations":[103],"into":[104],"small":[106],"fraction":[107],"stream's":[111],"effective":[112],"dimensions.":[113],"Our":[114],"experiments":[115],"reveal":[116],"this":[118],"dimensional":[122],"collapse":[123],"learned":[125,177],"rather":[126],"than":[127],"architectural,":[128],"is":[130,136],"largely":[131],"dissolved":[132],"when":[133],"structured":[134],"non-normality":[135],"removed.":[137],"further":[139],"topological":[143],"positioning":[144],"graph":[146],"communities":[147],"predicts":[148],"whether":[149],"amplifies":[152],"suppresses":[154],"them,":[155],"sign":[158],"coupling":[161],"determined":[162],"by":[163],"operator":[166],"type,":[167],"relationship":[169],"absent":[170],"at":[171],"initialization.":[172],"These":[173],"results":[174],"map":[175],"in":[180],"links":[183],"perturbation":[184],"propagation":[185],"compression":[187],"network's":[190],"functional":[191],"topology.":[192]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-16T00:00:00"}