{"id":"https://openalex.org/W7160954178","doi":"https://doi.org/10.48550/arxiv.2605.09011","title":"A Geometric Perspective on Next-Token Prediction in Large Language Models: Three Emerging Phases","display_name":"A Geometric Perspective on Next-Token Prediction in Large Language Models: Three Emerging Phases","publication_year":2026,"publication_date":"2026-05-09","ids":{"openalex":"https://openalex.org/W7160954178","doi":"https://doi.org/10.48550/arxiv.2605.09011"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.09011","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09011","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.09011","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054683542","display_name":"Gianfranco Lombardo","orcid":"https://orcid.org/0000-0003-1808-4487"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lombardo, Gianfranco","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094006444","display_name":"Giuseppe Trimigno","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Trimigno, Giuseppe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5090205116","display_name":"Stefano Cagnoni","orcid":"https://orcid.org/0000-0003-4669-512X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cagnoni, Stefano","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.19629999995231628,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.19629999995231628,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.09459999948740005,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.08429999649524689,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5809000134468079},{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.5422999858856201},{"id":"https://openalex.org/keywords/subspace-topology","display_name":"Subspace topology","score":0.5414000153541565},{"id":"https://openalex.org/keywords/linear-subspace","display_name":"Linear subspace","score":0.5015000104904175},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.48989999294281006},{"id":"https://openalex.org/keywords/affine-transformation","display_name":"Affine transformation","score":0.4657999873161316},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.45570001006126404},{"id":"https://openalex.org/keywords/superposition-principle","display_name":"Superposition principle","score":0.41679999232292175},{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.3930000066757202},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.36730000376701355}],"concepts":[{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5809000134468079},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.5422999858856201},{"id":"https://openalex.org/C32834561","wikidata":"https://www.wikidata.org/wiki/Q660730","display_name":"Subspace topology","level":2,"score":0.5414000153541565},{"id":"https://openalex.org/C12362212","wikidata":"https://www.wikidata.org/wiki/Q728435","display_name":"Linear subspace","level":2,"score":0.5015000104904175},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49470001459121704},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.48989999294281006},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.48750001192092896},{"id":"https://openalex.org/C92757383","wikidata":"https://www.wikidata.org/wiki/Q382497","display_name":"Affine transformation","level":2,"score":0.4657999873161316},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.45570001006126404},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.42480000853538513},{"id":"https://openalex.org/C27753989","wikidata":"https://www.wikidata.org/wiki/Q284885","display_name":"Superposition principle","level":2,"score":0.41679999232292175},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.3930000066757202},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3880999982357025},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.36730000376701355},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.3617999851703644},{"id":"https://openalex.org/C123403432","wikidata":"https://www.wikidata.org/wiki/Q654068","display_name":"Visibility","level":2,"score":0.3578000068664551},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.352400004863739},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3522999882698059},{"id":"https://openalex.org/C57493831","wikidata":"https://www.wikidata.org/wiki/Q3134666","display_name":"Projection (relational algebra)","level":2,"score":0.33219999074935913},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.32760000228881836},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.3151000142097473},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.3140000104904175},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.2962000072002411},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.28700000047683716},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.2849999964237213},{"id":"https://openalex.org/C529865628","wikidata":"https://www.wikidata.org/wiki/Q1790740","display_name":"Manifold (fluid mechanics)","level":2,"score":0.2815999984741211},{"id":"https://openalex.org/C207609745","wikidata":"https://www.wikidata.org/wiki/Q4944086","display_name":"Bootstrapping (finance)","level":2,"score":0.2782000005245209},{"id":"https://openalex.org/C2776650193","wikidata":"https://www.wikidata.org/wiki/Q264661","display_name":"Obstacle","level":2,"score":0.2766999900341034},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.27639999985694885},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.26750001311302185},{"id":"https://openalex.org/C14166107","wikidata":"https://www.wikidata.org/wiki/Q253829","display_name":"Net (polyhedron)","level":2,"score":0.26649999618530273},{"id":"https://openalex.org/C204241405","wikidata":"https://www.wikidata.org/wiki/Q461499","display_name":"Transformation (genetics)","level":3,"score":0.2637999951839447},{"id":"https://openalex.org/C109546454","wikidata":"https://www.wikidata.org/wiki/Q3798604","display_name":"Information geometry","level":4,"score":0.2632000148296356},{"id":"https://openalex.org/C175694140","wikidata":"https://www.wikidata.org/wiki/Q980329","display_name":"Orthographic projection","level":2,"score":0.26159998774528503},{"id":"https://openalex.org/C104065381","wikidata":"https://www.wikidata.org/wiki/Q1002535","display_name":"Geometric modeling","level":2,"score":0.2612999975681305},{"id":"https://openalex.org/C137635306","wikidata":"https://www.wikidata.org/wiki/Q182667","display_name":"Pareto principle","level":2,"score":0.2590000033378601},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.25760000944137573},{"id":"https://openalex.org/C110121322","wikidata":"https://www.wikidata.org/wiki/Q865811","display_name":"Distribution (mathematics)","level":2,"score":0.25040000677108765}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.09011","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09011","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.09011","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09011","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.42056676745414734,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,15,56],"investigate":[1],"the":[2,8,24,38,66,76,91,132,159,165,171,180,199,216,226,231,234,242,249],"geometry":[3],"of":[4,10,71,120,210,269],"predictive":[5,47,62],"information":[6,48],"across":[7,54,98,212],"layers":[9,188],"large":[11],"language":[12],"models":[13,141],"(LLMs).":[14],"repurpose":[16],"representation":[17],"lenses-learned":[18],"affine":[19],"maps":[20],"trained":[21],"to":[22,118,158,208,224],"predict":[23],"next":[25],"token":[26,201],"from":[27,116,142,206],"intermediate":[28],"residual":[29,78,160],"streams-as":[30],"geometric":[31,152],"diagnostic":[32],"tools.":[33],"Rather":[34],"than":[35],"asking":[36],"what":[37,163],"model":[39,259],"predicts":[40],"at":[41,58,136],"each":[42,59],"layer,":[43],"we":[44,149],"ask":[45],"where":[46],"resides":[49],"and":[50,86,112,128,146,177,186,254],"how":[51],"it":[52],"evolves":[53],"depth.":[55],"define":[57],"layer":[60],"a":[61,73,83,95,109,123,190,245],"readout":[63],"subspace":[64,70],"as":[65,94,203],"dominant":[67],"k-dimensional":[68],"singular":[69],"such":[72],"map":[74],"on":[75,90,170],"d-dimensional":[77],"stream":[79,161],"(where":[80],"k":[81,115],"is":[82,102,167,272],"resolution":[84],"parameter),":[85],"track":[87],"its":[88],"trajectory":[89],"Grassmann":[92],"manifold":[93],"similarity":[96],"profile":[97,101],"layers.":[99],"The":[100,266],"well":[103],"described":[104],"by":[105,275],"unimodal":[106],"distributions":[107],"exhibiting":[108],"rise,":[110],"near-plateau,":[111],"descent;":[113],"varying":[114],"1%":[117],"50%":[119],"d":[121],"traces":[122],"Pareto":[124],"frontier":[125],"between":[126],"visibility":[127],"energy":[129],"retention,":[130],"yet":[131],"same":[133],"structure":[134],"emerges":[135],"all":[137],"scales.":[138],"Across":[139],"eight":[140],"two":[143],"families":[144],"(Qwen2.5":[145],"OLMo2,":[147],"1B-32B),":[148],"identify":[150],"three":[151],"phases.":[153],"Updates":[154],"are":[155],"approximately":[156],"orthogonal":[157],"throughout;":[162],"distinguishes":[164],"phases":[166],"their":[168],"effect":[169],"effective":[172],"rank,":[173],"which":[174],"expands,":[175],"stabilizes,":[176],"concentrates.":[178],"In":[179,215,233],"first,":[181],"Seeding":[182],"Multiplexing,":[183],"feed-forward":[184],"memories":[185],"attention":[187],"seed":[189],"candidate":[191,205,227,276],"set":[192],"in":[193,195],"superposition":[194],"family-specific":[196],"proportions,":[197],"with":[198,248,258],"final":[200],"rising":[202],"leading":[204],"20%":[207],"35%":[209],"positions":[211],"this":[213],"phase.":[214],"second,":[217],"Hoisting":[218],"Overriding,":[219],"updates":[220,240],"override":[221],"existing":[222],"subspaces":[223],"concentrate":[225],"distribution":[228],"without":[229],"expanding":[230],"rank.":[232],"third,":[235],"Focal":[236],"Convergence,":[237],"high-energy":[238],"low-rank":[239],"write":[241],"winner":[243],"into":[244],"form":[246],"aligned":[247],"unembedding":[250],"direction.":[251],"Phases":[252],"1":[253],"3":[255],"grow":[256],"slowly":[257],"depth,":[260],"while":[261],"Phase":[262],"2":[263],"expands":[264],"linearly.":[265],"additional":[267],"capacity":[268],"deeper":[270],"LLMs":[271],"largely":[273],"absorbed":[274],"disambiguation.":[277]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-13T00:00:00"}