{"id":"https://openalex.org/W7204887350","doi":"https://doi.org/10.48550/arxiv.2608.29702","title":"A Hub of Short Rows Inflates Intrinsic Dimension Estimation of Token Embeddings","display_name":"A Hub of Short Rows Inflates Intrinsic Dimension Estimation of Token Embeddings","publication_year":2026,"publication_date":"2026-08-30","ids":{"openalex":"https://openalex.org/W7204887350","doi":"https://doi.org/10.48550/arxiv.2608.29702"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.29702","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.29702","pdf_url":"https://arxiv.org/pdf/2608.29702","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.29702","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5025340413","display_name":"Alexandre Quemy","orcid":"https://orcid.org/0000-0002-5865-6403","observed_orcids":["https://orcid.org/0000-0002-5865-6403"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Quemy, Alexandre","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5025340413"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9351000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9351000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.010099999606609344,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.009100000374019146,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/token-embeddings","display_name":"token embeddings","score":0.8289999961853027},{"id":"https://openalex.org/keywords/intrinsic-dimension","display_name":"intrinsic dimension","score":0.7170000076293945},{"id":"https://openalex.org/keywords/embedding-tables","display_name":"embedding tables","score":0.39500001072883606},{"id":"https://openalex.org/keywords/concentration-of-measure","display_name":"concentration of measure","score":0.7720000147819519},{"id":"https://openalex.org/keywords/heavy-tailed-distributions","display_name":"heavy-tailed distributions","score":0.7860000133514404},{"id":"https://openalex.org/keywords/pythia","display_name":"Pythia","score":0.6439999938011169},{"id":"https://openalex.org/keywords/language-model-embeddings","display_name":"language model embeddings","score":0.45100000500679016}],"concepts":[{"id":"https://openalex.org/C135598885","wikidata":"https://www.wikidata.org/wiki/Q1366302","display_name":"Row","level":2,"score":0.9433000087738037},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.7954000234603882},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.6868000030517578},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6554999947547913},{"id":"https://openalex.org/C45235069","wikidata":"https://www.wikidata.org/wiki/Q278425","display_name":"Table (database)","level":2,"score":0.6383000016212463},{"id":"https://openalex.org/C104140500","wikidata":"https://www.wikidata.org/wiki/Q2088159","display_name":"Row and column spaces","level":3,"score":0.6110000014305115},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5289000272750854},{"id":"https://openalex.org/C119322782","wikidata":"https://www.wikidata.org/wiki/Q2662236","display_name":"VC dimension","level":2,"score":0.499099999666214},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.4839000105857849},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.4611000120639801},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.4388999938964844},{"id":"https://openalex.org/C164866538","wikidata":"https://www.wikidata.org/wiki/Q367351","display_name":"Cluster (spacecraft)","level":2,"score":0.38119998574256897},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3634999990463257},{"id":"https://openalex.org/C2780551164","wikidata":"https://www.wikidata.org/wiki/Q2306599","display_name":"Column (typography)","level":3,"score":0.3476000130176544},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.33709999918937683},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.30979999899864197},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.2806999981403351}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.29702","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.29702","pdf_url":"https://arxiv.org/pdf/2608.29702","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.29702","doi":"https://doi.org/10.48550/arxiv.2608.29702","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.29702","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.29702","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.29702","pdf_url":"https://arxiv.org/pdf/2608.29702","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4415334639","https://openalex.org/W7135378992","https://openalex.org/W4416034278","https://openalex.org/W7133237026","https://openalex.org/W4416037580","https://openalex.org/W7221880709","https://openalex.org/W7165155381","https://openalex.org/W7203652301","https://openalex.org/W7196937537","https://openalex.org/W7124748148"],"abstract_inverted_index":{"A":[0],"token-embedding":[1,158],"table":[2,159,186],"holds":[3],"a":[4,31,59,68,79,83,114,133,135,198,218],"hub":[5,51,130,182,196,211],"of":[6,26,29,156,200,237,247],"short":[7],"rows":[8,52,138,229,245],"near":[9],"its":[10,45],"origin,":[11],"and":[12,108,127,171,215],"we":[13,100,213,240],"show":[14,175,241],"that":[15,151,176,203,212,242],"this":[16,177],"cluster":[17,38],"biases":[18],"what":[19,224],"nearest-neighbor":[20],"intrinsic-dimension":[21],"(ID)":[22,155],"estimators":[23,63],"report.":[24],"Because":[25],"the":[27,36,55,61,72,88,94,102,104,109,144,152,162,181,185,201,210,244,251],"concentration":[28],"measure,":[30],"token":[32,77],"is":[33,82,183,230],"closer":[34],"to":[35,40,113,122,141,167,190,226],"central":[37],"than":[39],"any":[41],"other":[42],"token,":[43],"so":[44],"first":[46],"two":[47],"neighbors":[48],"are":[49,139],"both":[50],"at":[53,78,192],"nearly":[54],"same":[56,252],"distance.":[57],"As":[58],"result,":[60],"ID":[62],"such":[64,124],"as":[65,125,132,217],"TwoNN":[66],"return":[67],"dimension":[69,81,111,154],"far":[70],"above":[71],"real":[73],"ID.":[74],"Measured":[75,86],"one":[76],"time,":[80],"heavy-tailed":[84],"distribution.":[85],"on":[87,208],"full":[89],"vocabulary,":[90],"it":[91],"grows":[92,160],"with":[93,161],"model's":[95],"parameter":[96,163],"count.":[97],"However,":[98],"when":[99,180],"remove":[101],"hub,":[103],"heavy":[105],"tail":[106],"disappears":[107,179],"measured":[110],"collapses":[112],"narrow":[115],"range":[116],"for":[117],"eleven":[118],"models,":[119],"from":[120,165],"GPT-2":[121],"models":[123],"K3":[126],"GLM-4.7.":[128],"The":[129,195],"acts":[131],"switch:":[134],"few":[136],"hundred":[137],"enough":[140],"fully":[142],"inflate":[143],"estimate.":[145],"We":[146,174],"reproduced":[147],"an":[148,235],"experiment":[149],"stating":[150],"intrinsic":[153],"Pythia's":[157],"count,":[164],"$27$":[166],"$122$":[168],"between":[169],"160M":[170],"12B":[172],"parameters.":[173],"result":[178],"removed:":[184],"then":[187],"reads":[188],"$10$":[189],"$17$":[191],"every":[193],"size.":[194],"contains":[197],"subset":[199],"population":[202],"under-trained-token":[204],"detectors":[205],"flag,":[206],"but":[207],"Pythia":[209],"detected":[214],"removed":[216],"whole":[219],"was":[220],"updated":[221],"during":[222],"training:":[223],"seem":[225],"characterize":[227],"these":[228],"simply":[231],"their":[232],"length,":[233],"not":[234],"absence":[236],"updates.":[238],"Finally,":[239],"normalizing":[243],"instead":[246],"removing":[248],"them":[249],"gives":[250],"lower":[253],"reading.":[254]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-09-02T00:00:00"}