{"id":"https://openalex.org/W7214735154","doi":"https://doi.org/10.48550/arxiv.2609.34731","title":"Information-Theoretic Analysis of Next-Token Prediction under Markovian Data","display_name":"Information-Theoretic Analysis of Next-Token Prediction under Markovian Data","publication_year":2026,"publication_date":"2026-09-28","ids":{"openalex":"https://openalex.org/W7214735154","doi":"https://doi.org/10.48550/arxiv.2609.34731"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.34731","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.34731","pdf_url":"https://arxiv.org/pdf/2609.34731","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.34731","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5046229948","display_name":"Masoud Kavian","orcid":"https://orcid.org/0009-0004-4038-8297","observed_orcids":["https://orcid.org/0009-0004-4038-8297"]},"institutions":[{"id":"https://openalex.org/I4210123571","display_name":"Huawei Technologies (France)","ror":"https://ror.org/02rbzf697","country_code":"FR","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210123571"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Kavian, Masoud","raw_affiliation_strings":["Mathematical and Algorithmic Science Laboratory , Huawei Paris Research Center , 92100 Boulogne-Billancourt , France,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mathematical and Algorithmic Science Laboratory , Huawei Paris Research Center , 92100 Boulogne-Billancourt , France,","institution_ids":["https://openalex.org/I4210123571"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014906072","display_name":"Abdellatif Zaidi","orcid":"https://orcid.org/0000-0003-2023-9476","observed_orcids":["https://orcid.org/0000-0003-2023-9476"]},"institutions":[{"id":"https://openalex.org/I4210123571","display_name":"Huawei Technologies (France)","ror":"https://ror.org/02rbzf697","country_code":"FR","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210123571"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Zaidi, Abdellatif","raw_affiliation_strings":["Mathematical and Algorithmic Science Laboratory , Huawei Paris Research Center , 92100 Boulogne-Billancourt , France,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mathematical and Algorithmic Science Laboratory , Huawei Paris Research Center , 92100 Boulogne-Billancourt , France,","institution_ids":["https://openalex.org/I4210123571"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077868963","display_name":"Milad Sefidgaran","orcid":"https://orcid.org/0000-0002-3576-9552","observed_orcids":["https://orcid.org/0000-0002-3576-9552"]},"institutions":[{"id":"https://openalex.org/I4210123571","display_name":"Huawei Technologies (France)","ror":"https://ror.org/02rbzf697","country_code":"FR","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210123571"]},{"id":"https://openalex.org/I4210152518","display_name":"Laboratoire d'Informatique Gaspard-Monge","ror":"https://ror.org/04t50yk91","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I142631665","https://openalex.org/I2800365227","https://openalex.org/I4210145102","https://openalex.org/I4210152518","https://openalex.org/I4210154111","https://openalex.org/I4210159245"]},{"id":"https://openalex.org/I4210154111","display_name":"Universit\u00e9 Gustave Eiffel","ror":"https://ror.org/03x42jk29","country_code":"FR","type":"education","lineage":["https://openalex.org/I2800365227","https://openalex.org/I4210154111"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Sefidgaran, Milad","raw_affiliation_strings":["Laboratoire d'Informatique Gaspard Monge , Universit\u00e9 Gustave Eiffel , 77420 Champs-sur- Marne , France","Mathematical and Algorithmic Science Laboratory , Huawei Paris Research Center , 92100 Boulogne-Billancourt , France,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Laboratoire d'Informatique Gaspard Monge , Universit\u00e9 Gustave Eiffel , 77420 Champs-sur- Marne , France","institution_ids":["https://openalex.org/I4210152518","https://openalex.org/I4210154111"]},{"raw_affiliation_string":"Mathematical and Algorithmic Science Laboratory , Huawei Paris Research Center , 92100 Boulogne-Billancourt , France,","institution_ids":["https://openalex.org/I4210123571"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.27149999141693115,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.27149999141693115,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.22460000216960907,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1932000070810318,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/next-token-prediction","display_name":"next-token prediction","score":0.996999979019165},{"id":"https://openalex.org/keywords/generalization-bounds","display_name":"generalization bounds","score":0.8980000019073486},{"id":"https://openalex.org/keywords/markov-processes","display_name":"Markov processes","score":0.7799999713897705},{"id":"https://openalex.org/keywords/time-dependence","display_name":"time dependence","score":0.6890000104904175},{"id":"https://openalex.org/keywords/mixing","display_name":"mixing","score":0.5960000157356262},{"id":"https://openalex.org/keywords/mutual-information","display_name":"mutual information","score":0.9070000052452087},{"id":"https://openalex.org/keywords/algorithmic-dependence","display_name":"algorithmic dependence","score":0.7979999780654907},{"id":"https://openalex.org/keywords/rate-distortion-theory","display_name":"rate-distortion theory","score":0.8299999833106995},{"id":"https://openalex.org/keywords/self-attention","display_name":"self-attention","score":0.8379999995231628},{"id":"https://openalex.org/keywords/context-length","display_name":"context length","score":0.7239999771118164},{"id":"https://openalex.org/keywords/time-series-forecasting","display_name":"time series forecasting","score":0.5210000276565552},{"id":"https://openalex.org/keywords/tinystories","display_name":"TinyStories","score":0.4429999887943268}],"concepts":[{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.7627000212669373},{"id":"https://openalex.org/C152139883","wikidata":"https://www.wikidata.org/wiki/Q252973","display_name":"Mutual information","level":2,"score":0.5612999796867371},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5407999753952026},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.5396999716758728},{"id":"https://openalex.org/C138777275","wikidata":"https://www.wikidata.org/wiki/Q6884054","display_name":"Mixing (physics)","level":2,"score":0.5297999978065491},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.520799994468689},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5177000164985657},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.47360000014305115},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.45980000495910645},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.438400000333786},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.41190001368522644},{"id":"https://openalex.org/C126780896","wikidata":"https://www.wikidata.org/wiki/Q899871","display_name":"Distortion (music)","level":4,"score":0.4059000015258789},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.3596000075340271},{"id":"https://openalex.org/C163836022","wikidata":"https://www.wikidata.org/wiki/Q6771326","display_name":"Markov model","level":3,"score":0.33719998598098755},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.3312000036239624},{"id":"https://openalex.org/C12426560","wikidata":"https://www.wikidata.org/wiki/Q189569","display_name":"Basis (linear algebra)","level":2,"score":0.31060001254081726},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3091999888420105},{"id":"https://openalex.org/C54907487","wikidata":"https://www.wikidata.org/wiki/Q7915688","display_name":"Variable-order Markov model","level":4,"score":0.30469998717308044},{"id":"https://openalex.org/C8272713","wikidata":"https://www.wikidata.org/wiki/Q176737","display_name":"Stochastic process","level":2,"score":0.28760001063346863},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.28060001134872437},{"id":"https://openalex.org/C2777027219","wikidata":"https://www.wikidata.org/wiki/Q1284190","display_name":"Constant (computer programming)","level":2,"score":0.27630001306533813},{"id":"https://openalex.org/C52622258","wikidata":"https://www.wikidata.org/wiki/Q131222","display_name":"Information theory","level":2,"score":0.2662999927997589}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.34731","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.34731","pdf_url":"https://arxiv.org/pdf/2609.34731","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.34731","doi":"https://doi.org/10.48550/arxiv.2609.34731","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.34731","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.34731","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.34731","pdf_url":"https://arxiv.org/pdf/2609.34731","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214735154.pdf","grobid_xml":"https://content.openalex.org/works/W7214735154.grobid-xml"},"referenced_works_count":7,"referenced_works":["https://openalex.org/W1780200830","https://openalex.org/W1865368880","https://openalex.org/W3172946450","https://openalex.org/W4376652984","https://openalex.org/W4386755420","https://openalex.org/W4403885568","https://openalex.org/W4411950007"],"related_works":["https://openalex.org/W7204173874","https://openalex.org/W4407759428","https://openalex.org/W4411950007","https://openalex.org/W2572743863","https://openalex.org/W7162806620","https://openalex.org/W7164040274","https://openalex.org/W3203779267","https://openalex.org/W6967343425","https://openalex.org/W2993189505","https://openalex.org/W7162333328"],"abstract_inverted_index":{"We":[0,14,79],"develop":[1],"an":[2,42,182],"information-theoretic":[3],"framework":[4],"for":[5,56,113,127,201],"generalization":[6,44,108,175],"in":[7,106],"next-token":[8,131],"prediction":[9],"under":[10],"temporally":[11],"dependent":[12],"data.":[13],"consider":[15],"independent":[16],"trajectories":[17],"generated":[18],"by":[19,28,35],"finite-memory":[20],"Markov":[21,57],"processes":[22],"and":[23,51,68,129,148,154,164],"distinguish":[24],"algorithmic":[25],"dependence,":[26,33],"quantified":[27],"mutual":[29,89],"information,":[30],"from":[31],"temporal":[32,69,149],"characterized":[34],"mixing.":[36,150],"For":[37,120],"cross-entropy":[38],"loss,":[39],"we":[40,123],"derive":[41,124],"expected":[43],"bound":[45,83],"using":[46],"the":[47,62,72,76,82,92,99,107,138,174],"Donsker--Varadhan":[48],"variational":[49],"representation":[50],"a":[52,85,103,198],"McDiarmid-type":[53],"concentration":[54],"inequality":[55],"chains.":[58],"A":[59,177],"refinement":[60],"captures":[61],"joint":[63],"effect":[64],"of":[65,75,140],"context":[66,141],"length":[67],"mixing":[70,73],"through":[71,84],"properties":[74],"history-state":[77],"process.":[78],"then":[80],"extend":[81],"rate--distortion":[86],"formulation,":[87],"replacing":[88],"information":[90,94],"with":[91,189],"minimum":[93],"rate":[95],"required":[96],"to":[97],"represent":[98],"learned":[100],"model":[101,143],"within":[102],"prescribed":[104],"distortion":[105],"gap,":[109],"yielding":[110],"informative":[111],"guarantees":[112],"deterministic":[114],"algorithms":[115],"over":[116],"continuous":[117],"hypothesis":[118],"spaces.":[119],"margin-based":[121],"prediction,":[122],"explicit":[125],"bounds":[126],"linear":[128],"self-attention":[130],"predictors":[132],"via":[133],"noisy":[134],"low-dimensional":[135],"compression,":[136],"revealing":[137],"roles":[139],"length,":[142],"complexity,":[144],"sample":[145],"size,":[146],"margin,":[147],"Experiments":[151],"on":[152],"TinyStories":[153],"ETTh2":[155,179],"show":[156],"that":[157],"longer":[158],"contexts":[159],"can":[160],"reduce":[161,169],"both":[162],"training":[163,170],"test":[165],"losses,":[166],"but":[167],"typically":[168],"loss":[171],"more,":[172],"enlarging":[173],"gap.":[176],"complementary":[178],"analysis":[180],"identifies":[181],"effective":[183],"predictive-memory":[184],"scale":[185],"near":[186],"24":[187],"hours,":[188],"no":[190],"statistically":[191],"supported":[192],"improvement":[193],"beyond":[194],"this":[195],"scale,":[196],"offering":[197],"plausible":[199],"explanation":[200],"test-performance":[202],"saturation":[203],"at":[204],"larger":[205],"contexts.":[206]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-30T00:00:00"}