{"id":"https://openalex.org/W7160905910","doi":"https://doi.org/10.48550/arxiv.2605.08505","title":"Scaling Limits of Long-Context Transformers","display_name":"Scaling Limits of Long-Context Transformers","publication_year":2026,"publication_date":"2026-05-08","ids":{"openalex":"https://openalex.org/W7160905910","doi":"https://doi.org/10.48550/arxiv.2605.08505"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2605.08505","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2605.08505","pdf_url":"https://arxiv.org/pdf/2605.08505","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2605.08505","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135980342","display_name":"Giuseppe Bruno","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I118564535","display_name":"University of Bern","ror":"https://ror.org/02k7v4d05","country_code":"CH","type":"education","lineage":["https://openalex.org/I118564535"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Bruno, Giuseppe","raw_affiliation_strings":["( GB) Departement Mathematik und Statistik , University of Bern , Sidlerstrasse 5 , 3012 Bern , Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"( GB) Departement Mathematik und Statistik , University of Bern , Sidlerstrasse 5 , 3012 Bern , Switzerland","institution_ids":["https://openalex.org/I118564535"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135986768","display_name":"Shi Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I118564535","display_name":"University of Bern","ror":"https://ror.org/02k7v4d05","country_code":"CH","type":"education","lineage":["https://openalex.org/I118564535"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Chen, Shi","raw_affiliation_strings":["( GB) Departement Mathematik und Statistik , University of Bern , Sidlerstrasse 5 , 3012 Bern , Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"( GB) Departement Mathematik und Statistik , University of Bern , Sidlerstrasse 5 , 3012 Bern , Switzerland","institution_ids":["https://openalex.org/I118564535"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135948208","display_name":"Zhengjiang Lin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I118564535","display_name":"University of Bern","ror":"https://ror.org/02k7v4d05","country_code":"CH","type":"education","lineage":["https://openalex.org/I118564535"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Lin, Zhengjiang","raw_affiliation_strings":["( GB) Departement Mathematik und Statistik , University of Bern , Sidlerstrasse 5 , 3012 Bern , Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"( GB) Departement Mathematik und Statistik , University of Bern , Sidlerstrasse 5 , 3012 Bern , Switzerland","institution_ids":["https://openalex.org/I118564535"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031031216","display_name":"Yury Polyanskiy","orcid":"https://orcid.org/0000-0002-2109-0979","observed_orcids":["https://orcid.org/0000-0002-2109-0979"]},"institutions":[{"id":"https://openalex.org/I118564535","display_name":"University of Bern","ror":"https://ror.org/02k7v4d05","country_code":"CH","type":"education","lineage":["https://openalex.org/I118564535"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Polyanskiy, Yury","raw_affiliation_strings":["( GB) Departement Mathematik und Statistik , University of Bern , Sidlerstrasse 5 , 3012 Bern , Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"( GB) Departement Mathematik und Statistik , University of Bern , Sidlerstrasse 5 , 3012 Bern , Switzerland","institution_ids":["https://openalex.org/I118564535"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001421646","display_name":"Philippe Rigollet","orcid":"https://orcid.org/0000-0002-1554-8912","observed_orcids":["https://orcid.org/0000-0002-1554-8912","https://orcid.org/0000-0002-0135-7162"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rigollet, Philippe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.24070000648498535,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.24070000648498535,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11716","display_name":"Random Matrices and Applications","score":0.15889999270439148,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.13930000364780426,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scaling-limits","display_name":"scaling limits","score":0.6340000033378601},{"id":"https://openalex.org/keywords/inverse-temperature","display_name":"inverse temperature","score":0.6679999828338623},{"id":"https://openalex.org/keywords/intrinsic-dimension","display_name":"intrinsic dimension","score":0.7360000014305115},{"id":"https://openalex.org/keywords/backward-heat-equation","display_name":"backward heat equation","score":0.753000020980835}],"concepts":[{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.6873999834060669},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.6492999792098999},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.6065000295639038},{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.5202999711036682},{"id":"https://openalex.org/C2778722038","wikidata":"https://www.wikidata.org/wiki/Q17030643","display_name":"Homogenization (climate)","level":3,"score":0.4805000126361847},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.4724000096321106},{"id":"https://openalex.org/C207467116","wikidata":"https://www.wikidata.org/wiki/Q4385666","display_name":"Inverse","level":2,"score":0.46380001306533813},{"id":"https://openalex.org/C38976095","wikidata":"https://www.wikidata.org/wiki/Q752641","display_name":"Asymmetry","level":2,"score":0.4512999951839447},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.430400013923645},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.42660000920295715},{"id":"https://openalex.org/C118419359","wikidata":"https://www.wikidata.org/wiki/Q682585","display_name":"Supercritical fluid","level":2,"score":0.3953000009059906},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.3822999894618988},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.3513000011444092},{"id":"https://openalex.org/C164154869","wikidata":"https://www.wikidata.org/wiki/Q901927","display_name":"Critical exponent","level":3,"score":0.3499000072479248},{"id":"https://openalex.org/C2780388253","wikidata":"https://www.wikidata.org/wiki/Q5421508","display_name":"Exponent","level":2,"score":0.34389999508857727},{"id":"https://openalex.org/C2781204021","wikidata":"https://www.wikidata.org/wiki/Q6497091","display_name":"Lattice (music)","level":2,"score":0.33559998869895935},{"id":"https://openalex.org/C64812099","wikidata":"https://www.wikidata.org/wiki/Q176604","display_name":"Random matrix","level":3,"score":0.3075000047683716},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.3041999936103821},{"id":"https://openalex.org/C24084028","wikidata":"https://www.wikidata.org/wiki/Q7429781","display_name":"Scaling limit","level":3,"score":0.29179999232292175},{"id":"https://openalex.org/C125112378","wikidata":"https://www.wikidata.org/wiki/Q176640","display_name":"Randomness","level":2,"score":0.287200003862381},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.2809000015258789},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.27619999647140503},{"id":"https://openalex.org/C45340560","wikidata":"https://www.wikidata.org/wiki/Q215382","display_name":"Disjoint sets","level":2,"score":0.2745000123977661},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.26820001006126404},{"id":"https://openalex.org/C171752962","wikidata":"https://www.wikidata.org/wiki/Q255166","display_name":"Kullback\u2013Leibler divergence","level":2,"score":0.26269999146461487},{"id":"https://openalex.org/C196298200","wikidata":"https://www.wikidata.org/wiki/Q577705","display_name":"Critical point (mathematics)","level":2,"score":0.2529999911785126}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2605.08505","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2605.08505","pdf_url":"https://arxiv.org/pdf/2605.08505","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2605.08505","doi":"https://doi.org/10.48550/arxiv.2605.08505","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08505","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2605.08505","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2605.08505","pdf_url":"https://arxiv.org/pdf/2605.08505","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7160905910.pdf","grobid_xml":"https://content.openalex.org/works/W7160905910.grobid-xml"},"referenced_works_count":7,"referenced_works":["https://openalex.org/W3015468748","https://openalex.org/W4285206226","https://openalex.org/W4387225871","https://openalex.org/W4392679398","https://openalex.org/W4407091930","https://openalex.org/W4417521252","https://openalex.org/W7128723349"],"related_works":["https://openalex.org/W7161190652","https://openalex.org/W4414977149","https://openalex.org/W7209311674","https://openalex.org/W7163829511","https://openalex.org/W4407091930","https://openalex.org/W4415330313","https://openalex.org/W4415798890","https://openalex.org/W6941297303","https://openalex.org/W6996611579","https://openalex.org/W7162039850"],"abstract_inverted_index":{"We":[0,57],"study":[1],"the":[2,21,24,28,34,39,53,60,70,74,85,101,105,111,124,167,174,182],"long-context":[3],"limit":[4],"of":[5,16,27,73,84,104,110,117,148],"softmax":[6],"self-attention":[7],"with":[8,23,133,177],"a":[9,13,119,128,140,145,158,187],"fixed":[10],"query":[11],"and":[12,87,109,137,157],"random":[14],"context":[15],"$n$":[17],"i.i.d.":[18],"keys":[19,95,150],"on":[20,96,166],"sphere":[22],"intrinsic":[25],"dimension":[26],"key":[29],"distribution":[30,76],"held":[31],"fixed,":[32],"viewing":[33],"inverse":[35],"temperature":[36],"$\u03b2_n$":[37],"as":[38],"scaling":[40],"parameter":[41],"that":[42,59],"decides":[43],"whether":[44],"attention":[45,107,112,183],"degenerates":[46],"into":[47],"uniform":[48,94],"averaging":[49],"or":[50],"collapses":[51],"onto":[52],"single":[54],"closest":[55,168],"key.":[56,169],"show":[58],"critical":[61,141],"scale":[62],"at":[63],"which":[64,123,144,162],"selectivity":[65],"emerges":[66],"is":[67,173],"determined":[68],"by":[69,81],"local":[71,129],"exponent":[72],"distance-to-query":[75],"near":[77],"zero":[78],"rather":[79],"than":[80],"global":[82],"features":[83],"context,":[86],"scales":[88],"like":[89],"$\u03b2_n^\\ast":[90],"\\asymp":[91],"n^{2/(d-1)}$":[92],"for":[93],"$\\mathbb{S}^{d-1}$.":[97],"Furthermore,":[98],"we":[99],"characterize":[100],"limiting":[102],"laws":[103],"ordered":[106],"weights":[108],"output":[113,125],"across":[114],"all":[115,163],"regimes":[116],"$\u03b2_n$:":[118],"subcritical":[120,175],"regime":[121,142,160],"in":[122,143,161],"reduces":[126],"to":[127],"average":[130],"around":[131],"$q$":[132],"explicit":[134],"deterministic":[135],"bias":[136],"Gaussian":[138],"fluctuations;":[139],"finite":[146],"collection":[147],"nearest":[149],"retains":[151],"macroscopic":[152],"mass":[153,164],"without":[154],"single-key":[155],"collapse;":[156],"supercritical":[159],"concentrates":[165],"Of":[170],"notable":[171],"interest":[172],"case":[176],"identity":[178],"value":[179],"matrix":[180],"where":[181],"map":[184],"approximately":[185],"implements":[186],"backward":[188],"heat":[189],"equation.":[190]},"counts_by_year":[{"year":2026,"cited_by_count":4}],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-05-13T00:00:00"}