{"id":"https://openalex.org/W7153191640","doi":"https://doi.org/10.48550/arxiv.2604.08118","title":"Initialisation Determines the Basin: Efficient Codebook Optimisation for Extreme LLM Quantization","display_name":"Initialisation Determines the Basin: Efficient Codebook Optimisation for Extreme LLM Quantization","publication_year":2026,"publication_date":"2026-04-09","ids":{"openalex":"https://openalex.org/W7153191640","doi":"https://doi.org/10.48550/arxiv.2604.08118"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.08118","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.08118","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.08118","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133359562","display_name":"Ian W. Kennedy","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kennedy, Ian W.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133364367","display_name":"Nafise Sadat Moosavi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Moosavi, Nafise Sadat","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10901","display_name":"Advanced Data Compression Techniques","score":0.41670000553131104,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10901","display_name":"Advanced Data Compression Techniques","score":0.41670000553131104,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10232","display_name":"Optical Network Technologies","score":0.039400000125169754,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10271","display_name":"Seismic Imaging and Inversion Techniques","score":0.0364999994635582,"subfield":{"id":"https://openalex.org/subfields/1908","display_name":"Geophysics"},"field":{"id":"https://openalex.org/fields/19","display_name":"Earth and Planetary Sciences"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/codebook","display_name":"Codebook","score":0.8859000205993652},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.6068000197410583},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.5430999994277954},{"id":"https://openalex.org/keywords/greedy-algorithm","display_name":"Greedy algorithm","score":0.412200003862381},{"id":"https://openalex.org/keywords/linde\u2013buzo\u2013gray-algorithm","display_name":"Linde\u2013Buzo\u2013Gray algorithm","score":0.41179999709129333},{"id":"https://openalex.org/keywords/data-compression","display_name":"Data compression","score":0.3571000099182129},{"id":"https://openalex.org/keywords/mahalanobis-distance","display_name":"Mahalanobis distance","score":0.3521000146865845},{"id":"https://openalex.org/keywords/enhanced-data-rates-for-gsm-evolution","display_name":"Enhanced Data Rates for GSM Evolution","score":0.34459999203681946}],"concepts":[{"id":"https://openalex.org/C127759330","wikidata":"https://www.wikidata.org/wiki/Q637416","display_name":"Codebook","level":2,"score":0.8859000205993652},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.6068000197410583},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5928999781608582},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.5430999994277954},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5293999910354614},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.43939998745918274},{"id":"https://openalex.org/C51823790","wikidata":"https://www.wikidata.org/wiki/Q504353","display_name":"Greedy algorithm","level":2,"score":0.412200003862381},{"id":"https://openalex.org/C93372532","wikidata":"https://www.wikidata.org/wiki/Q6552455","display_name":"Linde\u2013Buzo\u2013Gray algorithm","level":3,"score":0.41179999709129333},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.36410000920295715},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3630000054836273},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.3571000099182129},{"id":"https://openalex.org/C1921717","wikidata":"https://www.wikidata.org/wiki/Q1334846","display_name":"Mahalanobis distance","level":2,"score":0.3521000146865845},{"id":"https://openalex.org/C162307627","wikidata":"https://www.wikidata.org/wiki/Q204833","display_name":"Enhanced Data Rates for GSM Evolution","level":2,"score":0.34459999203681946},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.30140000581741333},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.28540000319480896},{"id":"https://openalex.org/C19889080","wikidata":"https://www.wikidata.org/wiki/Q2835852","display_name":"Beam search","level":3,"score":0.2768999934196472},{"id":"https://openalex.org/C84392682","wikidata":"https://www.wikidata.org/wiki/Q1952404","display_name":"Multilinear map","level":2,"score":0.2766999900341034},{"id":"https://openalex.org/C13481523","wikidata":"https://www.wikidata.org/wiki/Q412438","display_name":"Image compression","level":4,"score":0.27480000257492065},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2637999951839447},{"id":"https://openalex.org/C2988416141","wikidata":"https://www.wikidata.org/wiki/Q6031139","display_name":"Information loss","level":2,"score":0.26030001044273376},{"id":"https://openalex.org/C199833920","wikidata":"https://www.wikidata.org/wiki/Q612536","display_name":"Vector quantization","level":2,"score":0.25760000944137573},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.25099998712539673}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.08118","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.08118","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.08118","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.08118","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Additive":[0],"quantization":[1],"enables":[2],"extreme":[3,134],"LLM":[4],"compression":[5,93],"with":[6,25,127],"O(1)":[7],"lookup-table":[8],"dequantization,":[9],"making":[10],"it":[11,20],"attractive":[12],"for":[13],"edge":[14],"deployment.":[15],"Yet":[16],"at":[17,130,135],"2-bit":[18],"precision,":[19],"often":[21],"fails":[22],"catastrophically,":[23],"even":[24],"extensive":[26],"search":[27,53,95,167],"and":[28,54,77,80,97,116,168],"finetuning.":[29],"We":[30,59],"show":[31],"that":[32,50],"the":[33,44,64,72,118,124,153],"dominant":[34],"bottleneck":[35,125],"is":[36],"codebook":[37,78],"initialisation.":[38],"Greedy":[39],"sequential":[40],"initialisation":[41,86,140,163],"frequently":[42],"places":[43],"model":[45,160],"in":[46,158],"poor":[47,139],"optimisation":[48,156],"regions":[49],"subsequent":[51,166],"beam":[52],"PV-tuning":[55,115],"struggle":[56],"to":[57],"overcome.":[58],"analyse":[60],"this":[61],"behaviour":[62],"through":[63],"representational":[65],"ratio":[66],"\\r{ho}":[67],"=":[68],"N/KM,":[69],"which":[70],"characterises":[71],"relationship":[73],"between":[74],"weight":[75],"groups":[76],"capacity,":[79],"propose":[81],"OA-EM,":[82],"an":[83],"output-aware":[84],"EM":[85],"method":[87],"using":[88],"Hessian-weighted":[89],"Mahalanobis":[90],"distance.":[91],"Across":[92],"rates,":[94],"budgets,":[96],"three":[98],"architectures":[99],"(Llama":[100],"3.2":[101],"3B,":[102],"Llama":[103],"3.1":[104],"8B,":[105],"Qwen":[106],"2.5":[107],"3B),":[108],"OA-EM":[109],"consistently":[110],"produces":[111],"better":[112],"solutions":[113],"after":[114],"dominates":[117],"quality-compute":[119],"frontier.":[120],"The":[121],"severity":[122],"of":[123,146,155],"scales":[126],"\\r{ho}:":[128],"moderate":[129],"3":[131],"bpp":[132],"but":[133],"2":[136],"bpp,":[137],"where":[138,162],"can":[141,164],"degrade":[142],"perplexity":[143],"by":[144],"orders":[145],"magnitude.":[147],"More":[148],"broadly,":[149],"our":[150],"results":[151],"highlight":[152],"importance":[154],"geometry":[157],"compressed":[159],"spaces,":[161],"dominate":[165],"fine-tuning.":[169]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-11T00:00:00"}