{"id":"https://openalex.org/W7162298793","doi":"https://doi.org/10.48550/arxiv.2605.23901","title":"LLMs as Noisy Channels: A Shannon Perspective on Model Capacity and Scaling Laws","display_name":"LLMs as Noisy Channels: A Shannon Perspective on Model Capacity and Scaling Laws","publication_year":2026,"publication_date":"2026-05-22","ids":{"openalex":"https://openalex.org/W7162298793","doi":"https://doi.org/10.48550/arxiv.2605.23901"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.23901","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.23901","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.23901","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125060909","display_name":"Xu Ouyang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ouyang, Xu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048523092","display_name":"Deyi Liu","orcid":"https://orcid.org/0000-0002-3321-2232"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Deyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136897499","display_name":"Yuhang Cai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cai, Yuhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136933561","display_name":"Jing Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136978845","display_name":"Yuan Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Yuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136980945","display_name":"Chen Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136979934","display_name":"Thomas Hartvigsen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hartvigsen, Thomas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5082788622","display_name":"Yiyuan Ma","orcid":"https://orcid.org/0000-0001-7008-9221"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Yiyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12131","display_name":"Wireless Signal Modulation Classification","score":0.06159999966621399,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12131","display_name":"Wireless Signal Modulation Classification","score":0.06159999966621399,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.06120000034570694,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.05900000035762787,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.6983000040054321},{"id":"https://openalex.org/keywords/monotonic-function","display_name":"Monotonic function","score":0.5557000041007996},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.5252000093460083},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.39719998836517334},{"id":"https://openalex.org/keywords/gaussian","display_name":"Gaussian","score":0.38370001316070557},{"id":"https://openalex.org/keywords/information-theory","display_name":"Information theory","score":0.3686999976634979},{"id":"https://openalex.org/keywords/scaling-law","display_name":"Scaling law","score":0.3610999882221222},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.34299999475479126}],"concepts":[{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.6983000040054321},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5738999843597412},{"id":"https://openalex.org/C72169020","wikidata":"https://www.wikidata.org/wiki/Q194404","display_name":"Monotonic function","level":2,"score":0.5557000041007996},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.5252000093460083},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.39719998836517334},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.38370001316070557},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.37059998512268066},{"id":"https://openalex.org/C52622258","wikidata":"https://www.wikidata.org/wiki/Q131222","display_name":"Information theory","level":2,"score":0.3686999976634979},{"id":"https://openalex.org/C2988430800","wikidata":"https://www.wikidata.org/wiki/Q428971","display_name":"Scaling law","level":3,"score":0.3610999882221222},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34769999980926514},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.3458000123500824},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3434999883174896},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.34299999475479126},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3294000029563904},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3190000057220459},{"id":"https://openalex.org/C87040749","wikidata":"https://www.wikidata.org/wiki/Q428971","display_name":"Power law","level":2,"score":0.3125},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.301800012588501},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.29280000925064087},{"id":"https://openalex.org/C180137035","wikidata":"https://www.wikidata.org/wiki/Q1756194","display_name":"Shannon\u2013Hartley theorem","level":4,"score":0.28529998660087585},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.28060001134872437},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.27390000224113464},{"id":"https://openalex.org/C125932096","wikidata":"https://www.wikidata.org/wiki/Q205472","display_name":"Zipf's law","level":2,"score":0.26649999618530273},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.26159998774528503},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.25369998812675476},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.25279998779296875}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.23901","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.23901","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.23901","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.23901","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.5894917845726013}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Existing":[0],"scaling":[1,91,148],"laws":[2,149],"for":[3,89],"Large":[4],"Language":[5],"Models":[6],"(LLMs),":[7],"predominantly":[8],"monotonic":[9,110,192],"power":[10],"laws,":[11,153],"fail":[12],"to":[13,61,67,112,185],"explain":[14],"emerging":[15],"non-monotonic":[16],"phenomena":[17],"such":[18],"as":[19,45],"catastrophic":[20],"overtraining":[21],"and":[22,64,79,124,132,138,150,158],"quantization-induced":[23],"degradation,":[24],"where":[25],"performance":[26,114],"deteriorates":[27],"despite":[28],"increased":[29],"compute.":[30],"We":[31,116],"propose":[32],"the":[33,54,74,180],"Shannon":[34,87,142],"Scaling":[35,143],"Law,":[36],"a":[37,49,85,98,107],"unified":[38],"theoretical":[39],"framework":[40],"that":[41],"models":[42,174],"LLM":[43],"training":[44,65],"information":[46],"transmission":[47],"over":[48],"noisy":[50],"channel,":[51],"grounded":[52],"in":[53],"Shannon-Hartley":[55],"theorem.":[56],"By":[57],"mapping":[58],"model":[59,92,183],"parameters":[60],"channel":[62],"bandwidth":[63],"tokens":[66,187],"signal":[68,78],"power,":[69],"our":[70,118],"formulation":[71],"explicitly":[72],"captures":[73],"interaction":[75],"between":[76],"learning":[77],"intrinsic":[80],"noise.":[81],"This":[82],"perspective":[83],"reveals":[84],"fundamental":[86],"capacity":[88],"LLMs:":[90],"size":[93],"or":[94],"data":[95],"without":[96],"preserving":[97],"sufficient":[99],"signal-to-noise":[100],"ratio":[101],"(SNR)":[102],"inevitably":[103],"amplifies":[104],"noise,":[105,130],"inducing":[106],"transition":[108],"from":[109],"improvement":[111],"U-shaped":[113],"degradation.":[115],"validate":[117],"theory":[119],"through":[120],"experiments":[121],"on":[122,135,171],"Pythia":[123,173],"OLMo2":[125],"under":[126],"perturbations,":[127],"including":[128],"Gaussian":[129],"quantization":[131],"supervised":[133],"fine-tuning":[134],"math,":[136],"QA":[137],"code":[139],"tasks.":[140],"The":[141],"Law":[144],"consistently":[145],"outperforms":[146],"classical":[147],"recent":[151],"perturbation-aware":[152],"achieving":[154],"strong":[155],"$R^2$":[156],"scores":[157],"accurately":[159],"capturing":[160],"loss":[161],"basins":[162],"missed":[163],"by":[164],"prior":[165],"approaches.":[166],"It":[167],"also":[168],"extrapolates:":[169],"fitted":[170],"$\\leq$6.9B":[172],"with":[175],"$\\leq$180B":[176],"tokens,":[177],"it":[178],"predicts":[179],"unseen":[181],"12B":[182],"up":[184],"307B":[186],"at":[188],"pooled":[189],"$R^2{=}0.847$,":[190],"while":[191],"baselines":[193],"collapse.":[194]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-26T00:00:00"}