{"id":"https://openalex.org/W7161146531","doi":"https://doi.org/10.48550/arxiv.2605.13687","title":"A Hierarchical Language Model with Predictable Scaling Laws and Provable Benefits of Reasoning","display_name":"A Hierarchical Language Model with Predictable Scaling Laws and Provable Benefits of Reasoning","publication_year":2026,"publication_date":"2026-05-13","ids":{"openalex":"https://openalex.org/W7161146531","doi":"https://doi.org/10.48550/arxiv.2605.13687"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.13687","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.13687","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.13687","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037445477","display_name":"Jason Gaitonde","orcid":"https://orcid.org/0000-0001-6697-2572"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gaitonde, Jason","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003106042","display_name":"Frederic Koehler","orcid":"https://orcid.org/0000-0001-5220-9680"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Koehler, Frederic","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136106984","display_name":"Elchanan Mossel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mossel, Elchanan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136147013","display_name":"Joonhyung Shin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shin, Joonhyung","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5076570918","display_name":"Allan Sly","orcid":"https://orcid.org/0009-0004-8183-1910"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sly, Allan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.23800000548362732,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.23800000548362732,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12029","display_name":"DNA and Biological Computing","score":0.09809999912977219,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12090","display_name":"Language and cultural evolution","score":0.08299999684095383,"subfield":{"id":"https://openalex.org/subfields/3316","display_name":"Cultural Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/autoregressive-model","display_name":"Autoregressive model","score":0.6700000166893005},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.6507999897003174},{"id":"https://openalex.org/keywords/sublinear-function","display_name":"Sublinear function","score":0.5644000172615051},{"id":"https://openalex.org/keywords/upper-and-lower-bounds","display_name":"Upper and lower bounds","score":0.49079999327659607},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.41589999198913574},{"id":"https://openalex.org/keywords/sample-variance","display_name":"Sample variance","score":0.3935999870300293},{"id":"https://openalex.org/keywords/gaussian","display_name":"Gaussian","score":0.3465000092983246},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.3450999855995178}],"concepts":[{"id":"https://openalex.org/C159877910","wikidata":"https://www.wikidata.org/wiki/Q2202883","display_name":"Autoregressive model","level":2,"score":0.6700000166893005},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.6507999897003174},{"id":"https://openalex.org/C117160843","wikidata":"https://www.wikidata.org/wiki/Q338652","display_name":"Sublinear function","level":2,"score":0.5644000172615051},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49549999833106995},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.49079999327659607},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.45399999618530273},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.41589999198913574},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4025000035762787},{"id":"https://openalex.org/C2993021520","wikidata":"https://www.wikidata.org/wiki/Q175199","display_name":"Sample variance","level":3,"score":0.3935999870300293},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38999998569488525},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.35199999809265137},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.3465000092983246},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.3450999855995178},{"id":"https://openalex.org/C151376022","wikidata":"https://www.wikidata.org/wiki/Q168698","display_name":"Exponential function","level":2,"score":0.3176000118255615},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.3149999976158142},{"id":"https://openalex.org/C141513077","wikidata":"https://www.wikidata.org/wiki/Q378542","display_name":"Independent and identically distributed random variables","level":3,"score":0.31349998712539673},{"id":"https://openalex.org/C61326573","wikidata":"https://www.wikidata.org/wiki/Q1496376","display_name":"Gaussian process","level":3,"score":0.29580000042915344},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.2913999855518341},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.28999999165534973},{"id":"https://openalex.org/C2781023610","wikidata":"https://www.wikidata.org/wiki/Q17006304","display_name":"Burstiness","level":3,"score":0.28949999809265137},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.28380000591278076},{"id":"https://openalex.org/C166963901","wikidata":"https://www.wikidata.org/wiki/Q287251","display_name":"Kurtosis","level":2,"score":0.2825999855995178},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.26649999618530273},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.2660999894142151},{"id":"https://openalex.org/C4199805","wikidata":"https://www.wikidata.org/wiki/Q2725903","display_name":"Gaussian noise","level":2,"score":0.26159998774528503}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.13687","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.13687","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.13687","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.13687","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,201],"introduce":[1],"a":[2,13,55,77,114,229],"family":[3],"of":[4,23,38,49,72,97,113,153,232],"synthetic":[5,218],"languages":[6],"with":[7,51,144,149,184,213],"hierarchical":[8],"structure":[9],"--":[10,18,116,197],"generated":[11,99],"by":[12,76],"broadcast":[14,87,130],"process":[15],"on":[16,165,216],"trees":[17],"for":[19,69,123],"which":[20],"the":[21,36,73,85,95,98,104,120,128,136,154,166,194,204,208,217,220],"role":[22],"context":[24,52,105,167,233],"length":[25,53,168],"and":[26,107,207],"reasoning":[27],"in":[28,47,81,103,135],"autoregressive":[29,181],"generation":[30],"can":[31,190],"be":[32],"analyzed":[33],"precisely.":[34],"At":[35],"heart":[37],"our":[39,224],"analytic":[40],"approach":[41],"is":[42],"an":[43,161,180,198],"\\emph{exact":[44],"$k$-gram":[45],"ansatz}":[46],"place":[48],"transformers":[50,214],"$k$,":[54],"substitution":[56],"we":[57,64,92,177],"then":[58],"validate":[59],"empirically.":[60],"Using":[61],"this":[62],"ansatz":[63],"derive":[65],"explicit":[66],"asymptotic":[67,225],"predictions":[68,206,226],"distributional":[70],"statistics":[71],"sequences":[74,142],"produced":[75],"trained":[78,215,221],"model,":[79],"instantiated":[80],"two":[82],"settings.":[83],"For":[84,127],"\\emph{Ising":[86],"process}":[88,131],"(a":[89,132],"soft-constrained":[90],"language),":[91],"prove":[93,178],"that":[94,112,179],"variance":[96],"sum":[100],"scales":[101],"log-linearly":[102],"depth":[106],"its":[108],"kurtosis":[109],"converges":[110],"to":[111,170],"Gaussian":[115],"both":[117,203],"deviating":[118],"from":[119,193],"true":[121,195],"language":[122,196],"any":[124],"sublinear":[125],"context.":[126],"\\emph{coloring":[129],"hard-constrained":[133],"language)":[134],"freezing":[137],"regime,":[138],"bounded-context":[139],"autoregression":[140],"produces":[141],"that,":[143],"high":[145],"probability,":[146],"are":[147],"inconsistent":[148],"\\emph{any}":[150],"valid":[151],"coloring":[152],"underlying":[155],"tree.":[156],"Together":[157],"these":[158],"results":[159],"imply":[160],"$\u03a9(n)$":[162],"lower":[163],"bound":[164,211],"required":[169],"faithfully":[171],"sample":[172,191],"length-$n$":[173],"sequences.":[174],"In":[175],"contrast,":[176],"\\emph{reasoning}":[182],"model":[183],"only":[185],"$\u0398(\\log":[186],"n)$":[187],"working":[188],"memory":[189],"exactly":[192],"exponential":[199],"improvement.":[200],"confirm":[202],"lower-bound":[205],"reasoning-based":[209],"upper":[210],"empirically":[212],"language;":[219],"models":[222],"track":[223],"quantitatively":[227],"across":[228],"wide":[230],"range":[231],"sizes.":[234]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-15T00:00:00"}