{"id":"https://openalex.org/W4415743499","doi":"https://doi.org/10.1109/jstsp.2025.3626264","title":"An Information Theory of Compute-Optimal Size Scaling, Emergence, and Plateaus in Language Models","display_name":"An Information Theory of Compute-Optimal Size Scaling, Emergence, and Plateaus in Language Models","publication_year":2025,"publication_date":"2025-10-01","ids":{"openalex":"https://openalex.org/W4415743499","doi":"https://doi.org/10.1109/jstsp.2025.3626264"},"language":null,"primary_location":{"id":"doi:10.1109/jstsp.2025.3626264","is_oa":true,"landing_page_url":"https://doi.org/10.1109/jstsp.2025.3626264","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1109/jstsp.2025.3626264","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048493011","display_name":"Anuj K. Nayak","orcid":"https://orcid.org/0000-0002-7657-4348"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anuj K. Nayak","raw_affiliation_strings":["University of Illinois Urbana-Champaign, Urbana, IL, USA","University of Illinois Urbana-Champaign, USA"],"raw_orcid":"https://orcid.org/0000-0002-7657-4348","affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]},{"raw_affiliation_string":"University of Illinois Urbana-Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5065423139","display_name":"Lav R. Varshney","orcid":"https://orcid.org/0000-0003-2798-5308"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]},{"id":"https://openalex.org/I59553526","display_name":"Stony Brook University","ror":"https://ror.org/05qghxh33","country_code":"US","type":"education","lineage":["https://openalex.org/I59553526"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lav R. Varshney","raw_affiliation_strings":["University of Illinois Urbana-Champaign, Urbana, IL, USA","Stony Brook University, USA"],"raw_orcid":"https://orcid.org/0000-0003-2798-5308","affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]},{"raw_affiliation_string":"Stony Brook University, USA","institution_ids":["https://openalex.org/I59553526"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.188,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.94435202,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"19","issue":"7","first_page":"1338","last_page":"1348"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.11010000109672546,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.11010000109672546,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12090","display_name":"Language and cultural evolution","score":0.10840000212192535,"subfield":{"id":"https://openalex.org/subfields/3316","display_name":"Cultural Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.09969999641180038,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.6351000070571899},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.6316999793052673},{"id":"https://openalex.org/keywords/bipartite-graph","display_name":"Bipartite graph","score":0.5708000063896179},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.5188999772071838},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.41269999742507935},{"id":"https://openalex.org/keywords/random-graph","display_name":"Random graph","score":0.37779998779296875},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.37400001287460327},{"id":"https://openalex.org/keywords/analogy","display_name":"Analogy","score":0.36230000853538513},{"id":"https://openalex.org/keywords/information-theory","display_name":"Information theory","score":0.358599990606308}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6797999739646912},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.6351000070571899},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.6316999793052673},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.6294999718666077},{"id":"https://openalex.org/C197657726","wikidata":"https://www.wikidata.org/wiki/Q174733","display_name":"Bipartite graph","level":3,"score":0.5708000063896179},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.5188999772071838},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.41269999742507935},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3955000042915344},{"id":"https://openalex.org/C47458327","wikidata":"https://www.wikidata.org/wiki/Q910404","display_name":"Random graph","level":3,"score":0.37779998779296875},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.37400001287460327},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.367000013589859},{"id":"https://openalex.org/C521332185","wikidata":"https://www.wikidata.org/wiki/Q185816","display_name":"Analogy","level":2,"score":0.36230000853538513},{"id":"https://openalex.org/C52622258","wikidata":"https://www.wikidata.org/wiki/Q131222","display_name":"Information theory","level":2,"score":0.358599990606308},{"id":"https://openalex.org/C8272713","wikidata":"https://www.wikidata.org/wiki/Q176737","display_name":"Stochastic process","level":2,"score":0.35749998688697815},{"id":"https://openalex.org/C143587482","wikidata":"https://www.wikidata.org/wiki/Q1543216","display_name":"Iterative and incremental development","level":2,"score":0.34610000252723694},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.31380000710487366},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.31299999356269836},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.3107999861240387},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.3073999881744385},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.3068000078201294},{"id":"https://openalex.org/C67692717","wikidata":"https://www.wikidata.org/wiki/Q187444","display_name":"Low-density parity-check code","level":3,"score":0.30399999022483826},{"id":"https://openalex.org/C76969082","wikidata":"https://www.wikidata.org/wiki/Q486902","display_name":"Mathematical model","level":2,"score":0.3021000027656555},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.29989999532699585},{"id":"https://openalex.org/C159694833","wikidata":"https://www.wikidata.org/wiki/Q2321565","display_name":"Iterative method","level":2,"score":0.29170000553131104},{"id":"https://openalex.org/C88230418","wikidata":"https://www.wikidata.org/wiki/Q131476","display_name":"Graph theory","level":2,"score":0.28780001401901245},{"id":"https://openalex.org/C122203268","wikidata":"https://www.wikidata.org/wiki/Q5862903","display_name":"Probability theory","level":2,"score":0.2858999967575073},{"id":"https://openalex.org/C2779193601","wikidata":"https://www.wikidata.org/wiki/Q20026918","display_name":"Mathematical theory","level":2,"score":0.2791999876499176},{"id":"https://openalex.org/C87868495","wikidata":"https://www.wikidata.org/wiki/Q750843","display_name":"Information processing","level":2,"score":0.27730000019073486},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.26170000433921814},{"id":"https://openalex.org/C74672266","wikidata":"https://www.wikidata.org/wiki/Q815859","display_name":"Language acquisition","level":2,"score":0.2547000050544739},{"id":"https://openalex.org/C2777151079","wikidata":"https://www.wikidata.org/wiki/Q141160","display_name":"Parity (physics)","level":2,"score":0.2517000138759613},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2515999972820282}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/jstsp.2025.3626264","is_oa":true,"landing_page_url":"https://doi.org/10.1109/jstsp.2025.3626264","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1109/jstsp.2025.3626264","is_oa":true,"landing_page_url":"https://doi.org/10.1109/jstsp.2025.3626264","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G835545720","display_name":null,"funder_award_id":"PHY-2112890","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W1530042113","https://openalex.org/W1536930200","https://openalex.org/W1651818244","https://openalex.org/W1995875735","https://openalex.org/W2021898394","https://openalex.org/W2030405774","https://openalex.org/W2095882513","https://openalex.org/W2162346886","https://openalex.org/W2180809782","https://openalex.org/W2296073425","https://openalex.org/W3097149262","https://openalex.org/W3131305202","https://openalex.org/W4242044878","https://openalex.org/W4253011213","https://openalex.org/W4409310882"],"related_works":[],"abstract_inverted_index":{"Recent":[0],"empirical":[1],"studies":[2],"show":[3],"three":[4],"phenomena":[5],"with":[6],"increasing":[7],"size":[8,13,85,116],"of":[9,30,49,63,78,107,112,117],"language":[10,32,90,118],"models:":[11],"compute-optimal":[12,84],"scaling,":[14],"emergent":[15],"capabilities,":[16],"and":[17,110],"performance":[18,113],"plateauing.":[19],"We":[20,121],"present":[21],"a":[22,101],"simple":[23,102],"unified":[24],"mathematical":[25],"framework":[26],"to":[27,60],"explain":[28],"all":[29],"these":[31],"model":[33],"scaling":[34,76,86],"phenomena,":[35],"building":[36],"on":[37,74],"recent":[38],"skill-text":[39],"bipartite":[40],"graph":[41],"frameworks":[42],"for":[43,89,104],"semantic":[44],"learning.":[45],"Modeling":[46],"the":[47,83,115],"learning":[48],"concepts":[50],"from":[51,95],"texts":[52],"as":[53,114],"an":[54,58],"iterative":[55,61],"process":[56],"yields":[57],"analogy":[59],"decoding":[62],"low-density":[64],"parity":[65],"check":[66],"(LDPC)":[67],"codes":[68],"in":[69],"information":[70],"theory.":[71],"Thence,":[72],"drawing":[73],"finitesize":[75],"characterizations":[77],"LDPC":[79],"decoding,":[80],"we":[81,99],"derive":[82],"(Chinchilla":[87],"rule)":[88],"models.":[91],"Further,":[92],"using":[93],"tools":[94],"random":[96],"network":[97],"theory,":[98],"provide":[100],"explanation":[103],"both":[105],"emergence":[106],"complex":[108],"skills":[109],"plateauing":[111],"models":[119],"scale.":[120],"see":[122],"multiple":[123],"plateaus.":[124]},"counts_by_year":[{"year":2026,"cited_by_count":3}],"updated_date":"2026-01-01T23:40:50.289205","created_date":"2025-10-31T00:00:00"}