{"id":"https://openalex.org/W7127437944","doi":"https://doi.org/10.48550/arxiv.2602.02001","title":"Preserve-Then-Quantize: Balancing Rank Budgets for Quantization Error Reconstruction in LLMs","display_name":"Preserve-Then-Quantize: Balancing Rank Budgets for Quantization Error Reconstruction in LLMs","publication_year":2026,"publication_date":"2026-02-02","ids":{"openalex":"https://openalex.org/W7127437944","doi":"https://doi.org/10.48550/arxiv.2602.02001"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.02001","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5077647460","display_name":"Yoonjun Cho","orcid":"https://orcid.org/0000-0002-2186-6239"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cho, Yoonjun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085591636","display_name":"Dahyun Jeon","orcid":"https://orcid.org/0000-0003-1572-2981"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jeon, Dongjae","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124944494","display_name":"Soeun Kim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Soeun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124970666","display_name":"Moongyu Jeon","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jeon, Moongyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5049196468","display_name":"Albert No","orcid":"https://orcid.org/0000-0002-6346-4182"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"No, Albert","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.13502925,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10901","display_name":"Advanced Data Compression Techniques","score":0.3547999858856201,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10901","display_name":"Advanced Data Compression Techniques","score":0.3547999858856201,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.16019999980926514,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.07440000027418137,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.7991999983787537},{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.6108999848365784},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.5724999904632568},{"id":"https://openalex.org/keywords/perplexity","display_name":"Perplexity","score":0.5139999985694885},{"id":"https://openalex.org/keywords/round-off-error","display_name":"Round-off error","score":0.3995000123977661},{"id":"https://openalex.org/keywords/error-detection-and-correction","display_name":"Error detection and correction","score":0.3961000144481659},{"id":"https://openalex.org/keywords/subspace-topology","display_name":"Subspace topology","score":0.36340001225471497}],"concepts":[{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.7991999983787537},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.6108999848365784},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.5724999904632568},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5623000264167786},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5602999925613403},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.5139999985694885},{"id":"https://openalex.org/C61005703","wikidata":"https://www.wikidata.org/wiki/Q2145211","display_name":"Round-off error","level":2,"score":0.3995000123977661},{"id":"https://openalex.org/C103088060","wikidata":"https://www.wikidata.org/wiki/Q1062839","display_name":"Error detection and correction","level":2,"score":0.3961000144481659},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.3695000112056732},{"id":"https://openalex.org/C32834561","wikidata":"https://www.wikidata.org/wiki/Q660730","display_name":"Subspace topology","level":2,"score":0.36340001225471497},{"id":"https://openalex.org/C199833920","wikidata":"https://www.wikidata.org/wiki/Q612536","display_name":"Vector quantization","level":2,"score":0.3463999927043915},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.34439998865127563},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.33709999918937683},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.33340001106262207},{"id":"https://openalex.org/C93372532","wikidata":"https://www.wikidata.org/wiki/Q6552455","display_name":"Linde\u2013Buzo\u2013Gray algorithm","level":3,"score":0.32600000500679016},{"id":"https://openalex.org/C139945424","wikidata":"https://www.wikidata.org/wiki/Q1940696","display_name":"Mean squared error","level":2,"score":0.29120001196861267},{"id":"https://openalex.org/C167085575","wikidata":"https://www.wikidata.org/wiki/Q6803654","display_name":"Mean squared prediction error","level":2,"score":0.28760001063346863},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.26510000228881836},{"id":"https://openalex.org/C141379421","wikidata":"https://www.wikidata.org/wiki/Q6094427","display_name":"Iterative reconstruction","level":2,"score":0.26409998536109924}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.02001","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.02001","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.02001","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.02001","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.4472513198852539,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Quantization":[0,9],"Error":[1],"Reconstruction":[2,56],"(QER)":[3],"reduces":[4],"accuracy":[5],"loss":[6],"in":[7,139],"Post-Training":[8],"(PTQ)":[10],"by":[11,94],"approximating":[12],"weights":[13],"as":[14],"$\\mathbf{W}":[15],"\\approx":[16],"\\mathbf{Q}":[17],"+":[18,110],"\\mathbf{L}\\mathbf{R}$,":[19],"using":[20],"a":[21,58,88,143],"rank-$r$":[22],"correction":[23],"to":[24,35],"reconstruct":[25],"quantization":[26,48,137],"error.":[27],"Prior":[28],"methods":[29],"devote":[30],"the":[31,63,68,75,79],"full":[32],"rank":[33,81,102],"budget":[34],"error":[36,84,100],"reconstruction,":[37],"which":[38],"is":[39,156],"suboptimal":[40],"when":[41],"$\\mathbf{W}$":[42],"has":[43],"intrinsic":[44],"low-rank":[45],"structure":[46],"and":[47,77,98,119,136],"corrupts":[49],"dominant":[50],"directions.":[51,127],"We":[52,86,104],"propose":[53],"Structured":[54],"Residual":[55],"(SRR),":[57],"rank-allocation":[59],"framework":[60],"that":[61,107],"preserves":[62],"top-$k$":[64],"singular":[65],"subspace":[66],"of":[67],"activation-scaled":[69],"weight":[70],"before":[71],"quantization,":[72],"quantizes":[73],"only":[74],"residual,":[76],"uses":[78],"remaining":[80],"$r-k$":[82],"for":[83,91],"reconstruction.":[85],"derive":[87],"theory-guided":[89],"criterion":[90],"selecting":[92],"$k$":[93],"balancing":[95],"quantization-exposed":[96],"energy":[97],"unrecoverable":[99],"under":[101,150],"constraints.":[103],"further":[105],"show":[106],"resulting":[108],"$\\mathbf{Q}":[109],"\\mathbf{L}\\mathbf{R}$":[111],"parameterization":[112],"naturally":[113],"supports":[114],"Quantized":[115],"Parameter-Efficient":[116],"Fine-Tuning":[117],"(QPEFT),":[118],"stabilizes":[120],"fine-tuning":[121],"via":[122],"gradient":[123],"scaling":[124],"along":[125,141],"preserved":[126],"Experiments":[128],"demonstrate":[129],"consistent":[130],"perplexity":[131],"reductions":[132],"across":[133],"diverse":[134],"models":[135],"settings":[138],"PTQ,":[140],"with":[142],"5.9":[144],"percentage-point":[145],"average":[146],"gain":[147],"on":[148],"GLUE":[149],"2-bit":[151],"QPEFT.":[152],"The":[153],"project":[154],"page":[155],"available":[157],"at":[158],"https://ai-isl.github.io/srr.":[159]},"counts_by_year":[],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2026-02-04T00:00:00"}