{"id":"https://openalex.org/W7160905910","doi":"https://doi.org/10.48550/arxiv.2605.08505","title":"Scaling Limits of Long-Context Transformers","display_name":"Scaling Limits of Long-Context Transformers","publication_year":2026,"publication_date":"2026-05-08","ids":{"openalex":"https://openalex.org/W7160905910","doi":"https://doi.org/10.48550/arxiv.2605.08505"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.08505","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08505","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.08505","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135980342","display_name":"Giuseppe Bruno","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bruno, Giuseppe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135986768","display_name":"Shi Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Shi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135948208","display_name":"Zhengjiang Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Zhengjiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135995042","display_name":"Yury Polyanskiy","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Polyanskiy, Yury","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5001421646","display_name":"Philippe Rigollet","orcid":"https://orcid.org/0000-0002-1554-8912"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rigollet, Philippe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.1151999980211258,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.1151999980211258,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11804","display_name":"Quantum many-body systems","score":0.08829999715089798,"subfield":{"id":"https://openalex.org/subfields/3107","display_name":"Atomic and Molecular Physics, and Optics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11716","display_name":"Random Matrices and Applications","score":0.07530000060796738,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.6873999834060669},{"id":"https://openalex.org/keywords/softmax-function","display_name":"Softmax function","score":0.5202999711036682},{"id":"https://openalex.org/keywords/homogenization","display_name":"Homogenization (climate)","score":0.4805000126361847},{"id":"https://openalex.org/keywords/gaussian","display_name":"Gaussian","score":0.4724000096321106},{"id":"https://openalex.org/keywords/inverse","display_name":"Inverse","score":0.46380001306533813},{"id":"https://openalex.org/keywords/asymmetry","display_name":"Asymmetry","score":0.4512999951839447},{"id":"https://openalex.org/keywords/limiting","display_name":"Limiting","score":0.430400013923645},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.42660000920295715},{"id":"https://openalex.org/keywords/supercritical-fluid","display_name":"Supercritical fluid","score":0.3953000009059906}],"concepts":[{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.6873999834060669},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.6492999792098999},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.6065000295639038},{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.5202999711036682},{"id":"https://openalex.org/C2778722038","wikidata":"https://www.wikidata.org/wiki/Q17030643","display_name":"Homogenization (climate)","level":3,"score":0.4805000126361847},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.4724000096321106},{"id":"https://openalex.org/C207467116","wikidata":"https://www.wikidata.org/wiki/Q4385666","display_name":"Inverse","level":2,"score":0.46380001306533813},{"id":"https://openalex.org/C38976095","wikidata":"https://www.wikidata.org/wiki/Q752641","display_name":"Asymmetry","level":2,"score":0.4512999951839447},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.430400013923645},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.42660000920295715},{"id":"https://openalex.org/C118419359","wikidata":"https://www.wikidata.org/wiki/Q682585","display_name":"Supercritical fluid","level":2,"score":0.3953000009059906},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.3822999894618988},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.3513000011444092},{"id":"https://openalex.org/C164154869","wikidata":"https://www.wikidata.org/wiki/Q901927","display_name":"Critical exponent","level":3,"score":0.3499000072479248},{"id":"https://openalex.org/C2780388253","wikidata":"https://www.wikidata.org/wiki/Q5421508","display_name":"Exponent","level":2,"score":0.34389999508857727},{"id":"https://openalex.org/C2781204021","wikidata":"https://www.wikidata.org/wiki/Q6497091","display_name":"Lattice (music)","level":2,"score":0.33559998869895935},{"id":"https://openalex.org/C64812099","wikidata":"https://www.wikidata.org/wiki/Q176604","display_name":"Random matrix","level":3,"score":0.3075000047683716},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.3041999936103821},{"id":"https://openalex.org/C24084028","wikidata":"https://www.wikidata.org/wiki/Q7429781","display_name":"Scaling limit","level":3,"score":0.29179999232292175},{"id":"https://openalex.org/C125112378","wikidata":"https://www.wikidata.org/wiki/Q176640","display_name":"Randomness","level":2,"score":0.287200003862381},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.2809000015258789},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.27619999647140503},{"id":"https://openalex.org/C45340560","wikidata":"https://www.wikidata.org/wiki/Q215382","display_name":"Disjoint sets","level":2,"score":0.2745000123977661},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.26820001006126404},{"id":"https://openalex.org/C171752962","wikidata":"https://www.wikidata.org/wiki/Q255166","display_name":"Kullback\u2013Leibler divergence","level":2,"score":0.26269999146461487},{"id":"https://openalex.org/C196298200","wikidata":"https://www.wikidata.org/wiki/Q577705","display_name":"Critical point (mathematics)","level":2,"score":0.2529999911785126}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.08505","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08505","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.08505","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08505","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,47],"study":[1],"the":[2,21,24,29,43,50,60,64,75,91,95,101,114,157,164,172],"long-context":[3],"limit":[4],"of":[5,16,63,74,94,100,107,138],"softmax":[6],"self-attention":[7],"with":[8,123,167],"a":[9,13,109,118,130,135,148,177],"fixed":[10],"query":[11],"and":[12,77,99,127,147],"random":[14],"context":[15],"$n$":[17],"i.i.d.":[18],"keys":[19,85,140],"on":[20,86,156],"sphere,":[22],"viewing":[23],"inverse":[25],"temperature":[26],"$\u03b2_n$":[27],"as":[28],"scaling":[30],"parameter":[31],"that":[32,49],"decides":[33],"whether":[34],"attention":[35,97,102,173],"degenerates":[36],"into":[37],"uniform":[38,84],"averaging":[39],"or":[40],"collapses":[41],"onto":[42],"single":[44],"closest":[45,158],"key.":[46,159],"show":[48],"critical":[51,131],"scale":[52],"at":[53],"which":[54,113,134,152],"selectivity":[55],"emerges":[56],"is":[57,163],"determined":[58],"by":[59,71],"local":[61,119],"exponent":[62],"distance-to-query":[65],"distribution":[66],"near":[67],"zero":[68],"rather":[69],"than":[70],"global":[72],"features":[73],"context,":[76],"scales":[78],"like":[79],"$\u03b2_n^\\ast":[80],"\\asymp":[81],"n^{2/(d-1)}$":[82],"for":[83],"$\\mathbb{S}^{d-1}$.":[87],"Furthermore,":[88],"we":[89],"characterize":[90],"limiting":[92],"laws":[93],"ordered":[96],"weights":[98],"output":[103,115],"across":[104],"all":[105,153],"regimes":[106],"$\u03b2_n$:":[108],"subcritical":[110,165],"regime":[111,132,150],"in":[112,133,151],"reduces":[116],"to":[117],"average":[120],"around":[121],"$q$":[122],"explicit":[124],"deterministic":[125],"bias":[126],"Gaussian":[128],"fluctuations;":[129],"finite":[136],"collection":[137],"nearest":[139],"retains":[141],"macroscopic":[142],"mass":[143,154],"without":[144],"single-key":[145],"collapse;":[146],"supercritical":[149],"concentrates":[155],"Of":[160],"notable":[161],"interest":[162],"case":[166],"identity":[168],"value":[169],"matrix":[170],"where":[171],"map":[174],"approximately":[175],"implements":[176],"backward":[178],"heat":[179],"equation.":[180]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-13T00:00:00"}