{"id":"https://openalex.org/W7166010306","doi":"https://doi.org/10.48550/arxiv.2606.26749","title":"Structure Before Collapse: Transient semantic geometry in next-token prediction","display_name":"Structure Before Collapse: Transient semantic geometry in next-token prediction","alternate_titles":[],"publication_year":2026,"publication_date":"2026-06-25","ids":{"openalex":"https://openalex.org/W7166010306","doi":"https://doi.org/10.48550/arxiv.2606.26749"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2606.26749","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2606.26749","pdf_url":"https://arxiv.org/pdf/2606.26749","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W7166010306"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2606.26749","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Zhao, Yize","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Yize","raw_affiliation_strings":["Department of Electrical and Computer Engineering ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023621106","display_name":"Isabel Papadimitriou","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I141945490","display_name":"University of British Columbia","ror":"https://ror.org/03rmrcq20","country_code":"CA","type":"education","lineage":["https://openalex.org/I141945490"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Papadimitriou, Isabel","raw_affiliation_strings":["Department of Linguistics The University of British Columbia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Linguistics The University of British Columbia","institution_ids":["https://openalex.org/I141945490"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024812488","display_name":"Christos Thrampoulidis","orcid":"https://orcid.org/0000-0001-9053-9365","observed_orcids":["https://orcid.org/0000-0001-9053-9365"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Thrampoulidis, Christos","raw_affiliation_strings":["Department of Electrical and Computer Engineering ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering ,","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9452999830245972,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9452999830245972,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.023399999365210533,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.005200000014156103,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/neural-collapse","display_name":"neural collapse","score":0.8690000176429749},{"id":"https://openalex.org/keywords/next-token-prediction","display_name":"next-token prediction","score":0.8930000066757202},{"id":"https://openalex.org/keywords/language-models","display_name":"language models","score":0.7829999923706055},{"id":"https://openalex.org/keywords/semantic-geometry","display_name":"semantic geometry","score":0.859000027179718},{"id":"https://openalex.org/keywords/latent-semantic-structure","display_name":"latent semantic structure","score":0.6990000009536743},{"id":"https://openalex.org/keywords/representation-collapse","display_name":"representation collapse","score":0.5049999952316284},{"id":"https://openalex.org/keywords/phase-transitions","display_name":"phase transitions","score":0.7049999833106995},{"id":"https://openalex.org/keywords/emergent-structure","display_name":"emergent structure","score":0.460999995470047},{"id":"https://openalex.org/keywords/contextualized-embeddings","display_name":"contextual embeddings","score":0.3179999887943268}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6230000257492065},{"id":"https://openalex.org/C5274069","wikidata":"https://www.wikidata.org/wiki/Q2285707","display_name":"Categorical variable","level":2,"score":0.5964999794960022},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5938000082969666},{"id":"https://openalex.org/C112933361","wikidata":"https://www.wikidata.org/wiki/Q2845258","display_name":"Probabilistic latent semantic analysis","level":2,"score":0.5461000204086304},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5002999901771545},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.48739999532699585},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4803999960422516},{"id":"https://openalex.org/C170133592","wikidata":"https://www.wikidata.org/wiki/Q1806883","display_name":"Latent semantic analysis","level":2,"score":0.4641999900341034},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.4560999870300293},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.44859999418258667},{"id":"https://openalex.org/C130318100","wikidata":"https://www.wikidata.org/wiki/Q2268914","display_name":"Semantic similarity","level":2,"score":0.3977000117301941},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.39259999990463257},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.3747999966144562},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.31459999084472656},{"id":"https://openalex.org/C51167844","wikidata":"https://www.wikidata.org/wiki/Q4422623","display_name":"Latent variable","level":2,"score":0.3057999908924103},{"id":"https://openalex.org/C198942812","wikidata":"https://www.wikidata.org/wiki/Q496618","display_name":"Semantic property","level":2,"score":0.3021000027656555},{"id":"https://openalex.org/C36391188","wikidata":"https://www.wikidata.org/wiki/Q1939117","display_name":"Semantic change","level":2,"score":0.30160000920295715},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.29030001163482666},{"id":"https://openalex.org/C149106765","wikidata":"https://www.wikidata.org/wiki/Q1982453","display_name":"Denotation (semiotics)","level":3,"score":0.2770000100135803},{"id":"https://openalex.org/C65965080","wikidata":"https://www.wikidata.org/wiki/Q1806885","display_name":"Latent variable model","level":3,"score":0.27649998664855957},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.27140000462532043},{"id":"https://openalex.org/C143271835","wikidata":"https://www.wikidata.org/wiki/Q254515","display_name":"Similitude","level":2,"score":0.257999986410141},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.2538999915122986},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.25200000405311584}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2606.26749","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2606.26749","pdf_url":"https://arxiv.org/pdf/2606.26749","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2606.26749","doi":"https://doi.org/10.48550/arxiv.2606.26749","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.26749","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2606.26749","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2606.26749","pdf_url":"https://arxiv.org/pdf/2606.26749","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G202906059","display_name":null,"funder_award_id":"03677","funder_id":"https://openalex.org/F4320334593","funder_display_name":"Natural Sciences and Engineering Research Council of Canada"},{"id":"https://openalex.org/G4893171306","display_name":null,"funder_award_id":"ALLRP 581098-22","funder_id":"https://openalex.org/F4320334593","funder_display_name":"Natural Sciences and Engineering Research Council of Canada"}],"funders":[{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166010306.pdf","grobid_xml":"https://content.openalex.org/works/W7166010306.grobid-xml"},"referenced_works_count":69,"referenced_works":["https://openalex.org/W1488619990","https://openalex.org/W1490528330","https://openalex.org/W1614298861","https://openalex.org/W1615991656","https://openalex.org/W2048176942","https://openalex.org/W2057307785","https://openalex.org/W2111305191","https://openalex.org/W2125031621","https://openalex.org/W2141599568","https://openalex.org/W2153579005","https://openalex.org/W2160654481","https://openalex.org/W2250539671","https://openalex.org/W2518186251","https://openalex.org/W2946359678","https://openalex.org/W2946417913","https://openalex.org/W2950976310","https://openalex.org/W2963755427","https://openalex.org/W2970727289","https://openalex.org/W2999455489","https://openalex.org/W3017374003","https://openalex.org/W3065974826","https://openalex.org/W3104570641","https://openalex.org/W3109311007","https://openalex.org/W3147342393","https://openalex.org/W3159913668","https://openalex.org/W3169878837","https://openalex.org/W3198554905","https://openalex.org/W3202908475","https://openalex.org/W4296932880","https://openalex.org/W4386794314","https://openalex.org/W4386839891","https://openalex.org/W4387561538","https://openalex.org/W4388514540","https://openalex.org/W4389518851","https://openalex.org/W4392576577","https://openalex.org/W4398888709","https://openalex.org/W4399401478","https://openalex.org/W4402396053","https://openalex.org/W4402671179","https://openalex.org/W4402705760","https://openalex.org/W4403323947","https://openalex.org/W4403576725","https://openalex.org/W4404783413","https://openalex.org/W4404939297","https://openalex.org/W4407355147","https://openalex.org/W4410357454","https://openalex.org/W4412945484","https://openalex.org/W4415063744","https://openalex.org/W4415314067","https://openalex.org/W4415328688","https://openalex.org/W4415331749","https://openalex.org/W4415796798","https://openalex.org/W4415797094","https://openalex.org/W4415798616","https://openalex.org/W4415800591","https://openalex.org/W4415949701","https://openalex.org/W4416009630","https://openalex.org/W4416036519","https://openalex.org/W4416368807","https://openalex.org/W6891853381","https://openalex.org/W7083295743","https://openalex.org/W7103158775","https://openalex.org/W7127068361","https://openalex.org/W7129446325","https://openalex.org/W7133186543","https://openalex.org/W7135075179","https://openalex.org/W7140184047","https://openalex.org/W7196945805","https://openalex.org/W7196950407"],"related_works":["https://openalex.org/W7077144339","https://openalex.org/W4415314067","https://openalex.org/W4402705760","https://openalex.org/W7204940782","https://openalex.org/W7167837563","https://openalex.org/W7168169658","https://openalex.org/W4415800591","https://openalex.org/W4392576577","https://openalex.org/W7133944560","https://openalex.org/W7203634137"],"abstract_inverted_index":{"Neural":[0],"Collapse":[1],"predicts":[2],"that":[3,20,89,165,173],"balanced":[4],"one-hot":[5,51,78,131,161],"classification":[6],"pushes":[7],"model":[8,197,230],"representations":[9,174,206],"to":[10,59,100,130,159,184,224,231],"be":[11,101],"equally":[12,208],"far":[13],"from":[14],"each":[15],"other;":[16],"a":[17,37,81,221],"symmetric":[18,202],"configuration":[19],"depends":[21],"only":[22],"on":[23],"the":[24,33,53,77,87,90,96,106,196,200,225,233],"output":[25],"label":[26],"and":[27,172,194,219],"ignores":[28],"any":[29,134],"semantic":[30,124,154,166,235],"similarity":[31],"in":[32,62,93,105,170],"inputs.":[34],"This":[35,187],"creates":[36],"puzzle:":[38],"next-token":[39],"prediction":[40],"language":[41,82],"models":[42],"are":[43,157,207],"trained":[44],"predominantly":[45],"(as":[46],"context":[47,55],"length":[48],"increases)":[49],"with":[50,64,191],"labels:":[52],"same":[54],"is":[56,98,189],"very":[57],"unlikely":[58],"appear":[60],"twice":[61],"training":[63,79],"different":[65,138],"labels.":[66,162],"However,":[67],"they":[68],"clearly":[69],"learn":[70],"latent":[71,107,153],"structural":[72],"features.":[73],"That":[74],"is,":[75],"despite":[76,179],"regime,":[80],"model's":[83],"contextual":[84],"embeddings":[85],"represent":[86],"fact":[88],"next":[91],"word":[92],"''Mary":[94],"broke":[95],"___''":[97],"likely":[99],"filled":[102],"by":[103,176],"tokens":[104],"classes":[108],"of":[109],"a)":[110],"medium-sized,":[111],"b)":[112],"rigid,":[113],"c)":[114],"inanimate":[115],"nouns.":[116],"How":[117],"does":[118],"gradient":[119],"descent":[120],"find":[121,164],"such":[122],"categorical":[123],"structure":[125,188],"when":[126],"co-occurrence":[127],"statistics":[128],"collapse":[129],"sparsity,":[132],"eliminating":[133],"shared":[135,177],"next-tokens":[136],"among":[137],"contexts?":[139],"To":[140],"investigate":[141],"this":[142,212],"tension":[143],"we":[144],"identify":[145],"three":[146],"synthetic":[147],"controlled":[148],"settings":[149],"where":[150,204],"inputs":[151],"have":[152],"factors":[155],"but":[156],"mapped":[158],"distinct":[160],"We":[163,210],"geometry":[167],"emerges":[168],"early":[169],"training,":[171],"cluster":[175],"attributes":[178],"receiving":[180],"no":[181],"explicit":[182],"supervision":[183],"do":[185],"so.":[186],"transient:":[190],"sufficient":[192],"capacity":[193],"time,":[195],"eventually":[198],"reaches":[199],"predicted":[201],"state":[203],"all":[205],"separated.":[209],"study":[211],"phase":[213],"transition":[214],"through":[215],"Gram":[216],"matrix":[217],"analysis":[218],"propose":[220],"preliminary":[222],"modification":[223],"commonly":[226],"used":[227],"unconstrained":[228],"features":[229],"capture":[232],"emergent":[234],"geometry.":[236]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7166010306","counts_by_year":[{"year":2026,"cited_by_count":3}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2026-06-27T00:00:00"}
