{"id":"https://openalex.org/W7164910274","doi":"https://doi.org/10.48550/arxiv.2606.15789","title":"Approaching Shannon Bound with Lossless LLM Weight Compression","display_name":"Approaching Shannon Bound with Lossless LLM Weight Compression","publication_year":2026,"publication_date":"2026-06-14","ids":{"openalex":"https://openalex.org/W7164910274","doi":"https://doi.org/10.48550/arxiv.2606.15789"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.15789","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.15789","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.15789","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028775026","display_name":"Hongshi Tan","orcid":"https://orcid.org/0000-0002-3243-6875"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tan, Hongshi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138733130","display_name":"Yao Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138730762","display_name":"Gustavo Alonso","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alonso, Gustavo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138697676","display_name":"Weng-Fai Wong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wong, Weng-Fai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138690229","display_name":"Bingsheng He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Bingsheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.24979999661445618,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.24979999661445618,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.2175000011920929,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.09510000050067902,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/lossless-compression","display_name":"Lossless compression","score":0.8787999749183655},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.5257999897003174},{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.489300012588501},{"id":"https://openalex.org/keywords/data-compression","display_name":"Data compression","score":0.4772000014781952},{"id":"https://openalex.org/keywords/data-compression-ratio","display_name":"Data compression ratio","score":0.4625000059604645},{"id":"https://openalex.org/keywords/compression-ratio","display_name":"Compression ratio","score":0.4602000117301941},{"id":"https://openalex.org/keywords/terabyte","display_name":"Terabyte","score":0.459199994802475},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.4478999972343445},{"id":"https://openalex.org/keywords/entropy-encoding","display_name":"Entropy encoding","score":0.4180000126361847},{"id":"https://openalex.org/keywords/lossy-compression","display_name":"Lossy compression","score":0.38830000162124634}],"concepts":[{"id":"https://openalex.org/C81081738","wikidata":"https://www.wikidata.org/wiki/Q55542","display_name":"Lossless compression","level":3,"score":0.8787999749183655},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6848000288009644},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5770999789237976},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.5257999897003174},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.489300012588501},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.4772000014781952},{"id":"https://openalex.org/C94835093","wikidata":"https://www.wikidata.org/wiki/Q3113333","display_name":"Data compression ratio","level":5,"score":0.4625000059604645},{"id":"https://openalex.org/C25797200","wikidata":"https://www.wikidata.org/wiki/Q828137","display_name":"Compression ratio","level":3,"score":0.4602000117301941},{"id":"https://openalex.org/C199683683","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Terabyte","level":2,"score":0.459199994802475},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.4478999972343445},{"id":"https://openalex.org/C1769480","wikidata":"https://www.wikidata.org/wiki/Q1345239","display_name":"Entropy encoding","level":3,"score":0.4180000126361847},{"id":"https://openalex.org/C165021410","wikidata":"https://www.wikidata.org/wiki/Q55564","display_name":"Lossy compression","level":2,"score":0.38830000162124634},{"id":"https://openalex.org/C115051666","wikidata":"https://www.wikidata.org/wiki/Q6522493","display_name":"Ranging","level":2,"score":0.3750999867916107},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.3749000132083893},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.3741999864578247},{"id":"https://openalex.org/C125112378","wikidata":"https://www.wikidata.org/wiki/Q176640","display_name":"Randomness","level":2,"score":0.3659999966621399},{"id":"https://openalex.org/C2776029614","wikidata":"https://www.wikidata.org/wiki/Q1146367","display_name":"File size","level":2,"score":0.35580000281333923},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.3467000126838684},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.33570000529289246},{"id":"https://openalex.org/C153338461","wikidata":"https://www.wikidata.org/wiki/Q2651","display_name":"Arithmetic coding","level":4,"score":0.329800009727478},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.3287999927997589},{"id":"https://openalex.org/C13481523","wikidata":"https://www.wikidata.org/wiki/Q412438","display_name":"Image compression","level":4,"score":0.32519999146461487},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.32359999418258667},{"id":"https://openalex.org/C162932704","wikidata":"https://www.wikidata.org/wiki/Q1058791","display_name":"Synchronizing","level":3,"score":0.3203999996185303},{"id":"https://openalex.org/C182407805","wikidata":"https://www.wikidata.org/wiki/Q2626534","display_name":"Substring","level":3,"score":0.31380000710487366},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.30630001425743103},{"id":"https://openalex.org/C131097465","wikidata":"https://www.wikidata.org/wiki/Q178898","display_name":"Gas compressor","level":2,"score":0.29580000042915344},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.29089999198913574},{"id":"https://openalex.org/C171752962","wikidata":"https://www.wikidata.org/wiki/Q255166","display_name":"Kullback\u2013Leibler divergence","level":2,"score":0.2791000008583069},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.27300000190734863},{"id":"https://openalex.org/C101722063","wikidata":"https://www.wikidata.org/wiki/Q218825","display_name":"Random access","level":2,"score":0.26170000433921814},{"id":"https://openalex.org/C2776545253","wikidata":"https://www.wikidata.org/wiki/Q5464292","display_name":"Folding (DSP implementation)","level":2,"score":0.2590000033378601},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.25780001282691956}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.15789","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.15789","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.15789","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.15789","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.5208203196525574,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,47],"(LLMs)":[3],"now":[4],"scale":[5],"to":[6,50,59,87,174,180,191,196,201,216],"trillions":[7],"of":[8,124,134,142,170],"parameters,":[9],"driving":[10],"weight":[11,98],"storage":[12],"into":[13,155],"the":[14,120,135,156,166,184],"terabyte":[15],"regime":[16],"and":[17,53,61,207],"creating":[18],"an":[19],"acute":[20],"mismatch":[21],"with":[22,119,160],"GPU":[23,125],"memory":[24],"capacity.":[25],"Although":[26],"lossless":[27,108,203],"compression":[28,204],"is":[29,81,92,152],"widely":[30],"effective":[31,79],"in":[32,38],"other":[33],"domains,":[34],"it":[35],"remains":[36],"underutilized":[37],"LLM":[39,66,143],"systems.":[40],"Through":[41],"a":[42,88,105,139],"comprehensive":[43],"entropy":[44,80],"study":[45],"across":[46,138],"from":[48,57,172,189],"1.5B":[49],"405B":[51],"parameters":[52],"numeric":[54],"formats":[55],"ranging":[56],"bf16":[58],"int4":[60],"AWQ/SQ8,":[62],"we":[63,103],"find":[64],"that":[65,85,116,147],"weights":[67],"contain":[68],"far":[69],"less":[70],"intrinsic":[71],"randomness":[72],"than":[73],"their":[74,78],"stored":[75],"bitwidth":[76],"implies,":[77],"2-10x":[82],"lower,":[83],"indicating":[84],"up":[86,179,195,215],"10x":[89],"footprint":[90],"reduction":[91],"theoretically":[93],"achievable":[94],"without":[95],"altering":[96],"any":[97],"values.":[99],"Leveraging":[100],"this":[101],"insight,":[102],"introduce":[104],"tile-level,":[106],"on-the-fly":[107],"decompression":[109],"framework":[110,159],"based":[111],"on":[112],"Asymmetric":[113],"Numeral":[114],"Systems":[115],"aligns":[117],"decoding":[118],"GEMM":[121],"tiling":[122],"pattern":[123],"inference.":[126],"Our":[127],"design":[128,210],"achieves":[129],"bit-rates":[130],"within":[131],"0.01-0.1":[132],"bits":[133],"Shannon":[136],"limit":[137],"wide":[140],"range":[141],"numerical":[144],"formats,":[145],"demonstrating":[146],"nearly":[148],"all":[149],"statistical":[150],"redundancy":[151],"eliminated.":[153],"Integrated":[154],"SGLang":[157],"serving":[158],"multi-GPU":[161],"support,":[162],"our":[163,209],"approach":[164],"increases":[165,188],"maximum":[167],"batch":[168,186],"size":[169,187],"Qwen-14B":[171],"47":[173],"75,":[175],"improving":[176],"throughput":[177,198,213],"by":[178,214],"1.2x.":[181],"On":[182],"Mixtral-176B,":[183],"feasible":[185],"20":[190],"95":[192],"(4.8x),":[193],"yielding":[194],"1.6x":[197],"improvement.":[199],"Compared":[200],"state-of-the-art":[202],"approaches":[205],"NeuZip":[206],"DFloat11,":[208],"further":[211],"improves":[212],"11x.":[217]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-17T00:00:00"}