{"id":"https://openalex.org/W4415314067","doi":"https://doi.org/10.48550/arxiv.2505.08348","title":"Geometry of Semantics in Next-Token Prediction: How Optimization Implicitly Organizes Linguistic Representations","display_name":"Geometry of Semantics in Next-Token Prediction: How Optimization Implicitly Organizes Linguistic Representations","publication_year":2025,"publication_date":"2025-05-13","ids":{"openalex":"https://openalex.org/W4415314067","doi":"https://doi.org/10.48550/arxiv.2505.08348"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2505.08348","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.08348","pdf_url":"https://arxiv.org/pdf/2505.08348","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2505.08348","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Zhao, Yize","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Yize","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5024812488","display_name":"Christos Thrampoulidis","orcid":"https://orcid.org/0000-0001-9053-9365","observed_orcids":["https://orcid.org/0000-0001-9053-9365"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Thrampoulidis, Christos","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/next-token-prediction","display_name":"next-token prediction","score":0.9919999837875366},{"id":"https://openalex.org/keywords/language-models","display_name":"language models","score":0.7080000042915344},{"id":"https://openalex.org/keywords/singular-value-decomposition","display_name":"singular value decomposition","score":0.6970000267028809},{"id":"https://openalex.org/keywords/latent-semantic-structure","display_name":"latent semantic structure","score":0.5},{"id":"https://openalex.org/keywords/distributional-semantics","display_name":"distributional semantics","score":0.6320000290870667},{"id":"https://openalex.org/keywords/neural-collapse","display_name":"neural collapse","score":0.8920000195503235},{"id":"https://openalex.org/keywords/semantic-hierarchy","display_name":"semantic hierarchy","score":0.722000002861023},{"id":"https://openalex.org/keywords/word-embeddings","display_name":"word embeddings","score":0.5920000076293945},{"id":"https://openalex.org/keywords/context-embedding","display_name":"context embedding","score":0.8569999933242798},{"id":"https://openalex.org/keywords/gradient-based-optimization","display_name":"gradient-based optimization","score":0.7210000157356262},{"id":"https://openalex.org/keywords/grammatical-categories","display_name":"grammatical categories","score":0.6499999761581421}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6814000010490417},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.6633999943733215},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5861999988555908},{"id":"https://openalex.org/C22789450","wikidata":"https://www.wikidata.org/wiki/Q420904","display_name":"Singular value decomposition","level":2,"score":0.5504000186920166},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5419999957084656},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5302000045776367},{"id":"https://openalex.org/C31170391","wikidata":"https://www.wikidata.org/wiki/Q188619","display_name":"Hierarchy","level":2,"score":0.526199996471405},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.44589999318122864},{"id":"https://openalex.org/C2778828372","wikidata":"https://www.wikidata.org/wiki/Q5283209","display_name":"Distributional semantics","level":3,"score":0.44179999828338623},{"id":"https://openalex.org/C171018156","wikidata":"https://www.wikidata.org/wiki/Q7370306","display_name":"Rotation formalisms in three dimensions","level":2,"score":0.43389999866485596},{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.4124999940395355},{"id":"https://openalex.org/C130318100","wikidata":"https://www.wikidata.org/wiki/Q2268914","display_name":"Semantic similarity","level":2,"score":0.4077000021934509},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.4077000021934509},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.39340001344680786},{"id":"https://openalex.org/C139676723","wikidata":"https://www.wikidata.org/wiki/Q1193832","display_name":"Sign (mathematics)","level":2,"score":0.3824000060558319},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.3799000084400177},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3537999987602234},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.35179999470710754},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.3434000015258789},{"id":"https://openalex.org/C202708506","wikidata":"https://www.wikidata.org/wiki/Q7449050","display_name":"Semantic compression","level":5,"score":0.33219999074935913},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.3222000002861023},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.31029999256134033},{"id":"https://openalex.org/C42355184","wikidata":"https://www.wikidata.org/wiki/Q1361088","display_name":"Matrix decomposition","level":3,"score":0.30070000886917114},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2969000041484833},{"id":"https://openalex.org/C511149849","wikidata":"https://www.wikidata.org/wiki/Q7449051","display_name":"Semantic computing","level":3,"score":0.2915000021457672},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.28220000863075256},{"id":"https://openalex.org/C187834632","wikidata":"https://www.wikidata.org/wiki/Q188804","display_name":"Factorization","level":2,"score":0.2709999978542328},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.26339998841285706},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2614000141620636},{"id":"https://openalex.org/C2780276568","wikidata":"https://www.wikidata.org/wiki/Q191928","display_name":"Polysemy","level":2,"score":0.2599000036716461},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.2556999921798706},{"id":"https://openalex.org/C98954769","wikidata":"https://www.wikidata.org/wiki/Q1759657","display_name":"Lexical semantics","level":3,"score":0.2522999942302704}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2505.08348","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.08348","pdf_url":"https://arxiv.org/pdf/2505.08348","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2505.08348","doi":"https://doi.org/10.48550/arxiv.2505.08348","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2505.08348","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2505.08348","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.08348","pdf_url":"https://arxiv.org/pdf/2505.08348","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,79,121],"investigate":[1],"how":[2,159],"next-token":[3],"prediction":[4],"(NTP)":[5],"optimization":[6,161],"leads":[7],"language":[8,130],"models":[9,35,52],"to":[10,36,64,84,116],"extract":[11],"and":[12,26,60,128,143,154,168],"organize":[13],"semantic":[14,73,96,119,134,173],"structure":[15],"from":[16],"text.":[17],"Our":[18,148],"analysis,":[19],"based":[20],"on":[21,125],"a":[22,38,94,110],"tractable":[23],"mathematical":[24],"model":[25],"controlled":[27],"synthetic":[28,126],"data,":[29],"reveals":[30],"that":[31,81,112,171],"NTP":[32],"implicitly":[33,162],"guides":[34],"factor":[37],"centered":[39],"support":[40],"matrix":[41,166],"encoding":[42,71],"context-to-next-token":[43],"co-occurrence":[44],"patterns":[45],"via":[46],"singular":[47,69,86],"value":[48],"decomposition":[49],"(SVD).":[50],"While":[51],"never":[53],"explicitly":[54],"construct":[55],"this":[56],"matrix,":[57],"learned":[58,89],"word":[59],"context":[61],"embeddings":[62],"converge":[63],"its":[65],"SVD":[66],"factors,":[67],"with":[68],"vectors":[70],"latent":[72],"concepts":[74,82],"through":[75],"their":[76],"sign":[77],"patterns.":[78],"demonstrate":[80],"corresponding":[83],"larger":[85],"values":[87],"are":[88],"earlier":[90],"during":[91],"training,":[92],"yielding":[93],"natural":[95],"hierarchy":[97],"where":[98],"broad":[99],"categories":[100],"emerge":[101],"before":[102],"fine-grained":[103],"ones.":[104],"This":[105],"insight":[106],"motivates":[107],"orthant-based":[108],"clustering,":[109],"method":[111,170],"combines":[113],"concept":[114],"signs":[115],"identify":[117],"interpretable":[118],"categories.":[120],"validate":[122],"our":[123],"findings":[124],"datasets":[127],"pretrained":[129],"models,":[131],"recovering":[132],"diverse":[133],"structures":[135],"such":[136],"as":[137],"grammatical":[138],"categories,":[139],"named":[140],"entity":[141],"types,":[142],"topical":[144],"distinctions":[145],"(medical,":[146],"entertainment).":[147],"work":[149],"bridges":[150],"classical":[151],"distributional":[152],"semantics":[153],"neural":[155],"collapse":[156],"geometry,":[157],"characterizing":[158],"gradient-based":[160],"determines":[163],"both":[164],"the":[165],"representation":[167],"factorization":[169],"encode":[172],"structure.":[174]},"counts_by_year":[],"updated_date":"2026-10-14T16:39:01.820549","created_date":"2025-10-18T00:00:00"}