{"id":"https://openalex.org/W7128773446","doi":"https://doi.org/10.48550/arxiv.2602.11184","title":"KBVQ-MoE: KLT-guided SVD with Bias-Corrected Vector Quantization for MoE Large Language Models","display_name":"KBVQ-MoE: KLT-guided SVD with Bias-Corrected Vector Quantization for MoE Large Language Models","publication_year":2026,"publication_date":"2026-01-30","ids":{"openalex":"https://openalex.org/W7128773446","doi":"https://doi.org/10.48550/arxiv.2602.11184"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.11184","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125989007","display_name":"Zukang Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Zukang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125940974","display_name":"Zhixiong Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Zhixiong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125949227","display_name":"Xing Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Xing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chen, Zhixuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Zhixuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5125938233","display_name":"Dawei Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Dawei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.18866571,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.2076999992132187,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.2076999992132187,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.1648000031709671,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.057999998331069946,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/vector-quantization","display_name":"Vector quantization","score":0.8646000027656555},{"id":"https://openalex.org/keywords/codebook","display_name":"Codebook","score":0.7178999781608582},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.6579999923706055},{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.6207000017166138},{"id":"https://openalex.org/keywords/linde\u2013buzo\u2013gray-algorithm","display_name":"Linde\u2013Buzo\u2013Gray algorithm","score":0.6014000177383423},{"id":"https://openalex.org/keywords/affine-transformation","display_name":"Affine transformation","score":0.569599986076355},{"id":"https://openalex.org/keywords/singular-value-decomposition","display_name":"Singular value decomposition","score":0.4447000026702881}],"concepts":[{"id":"https://openalex.org/C199833920","wikidata":"https://www.wikidata.org/wiki/Q612536","display_name":"Vector quantization","level":2,"score":0.8646000027656555},{"id":"https://openalex.org/C127759330","wikidata":"https://www.wikidata.org/wiki/Q637416","display_name":"Codebook","level":2,"score":0.7178999781608582},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.6579999923706055},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.6207000017166138},{"id":"https://openalex.org/C93372532","wikidata":"https://www.wikidata.org/wiki/Q6552455","display_name":"Linde\u2013Buzo\u2013Gray algorithm","level":3,"score":0.6014000177383423},{"id":"https://openalex.org/C92757383","wikidata":"https://www.wikidata.org/wiki/Q382497","display_name":"Affine transformation","level":2,"score":0.569599986076355},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5382999777793884},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4659999907016754},{"id":"https://openalex.org/C22789450","wikidata":"https://www.wikidata.org/wiki/Q420904","display_name":"Singular value decomposition","level":2,"score":0.4447000026702881},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41819998621940613},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3828999996185303},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.36480000615119934},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.362199991941452},{"id":"https://openalex.org/C40567965","wikidata":"https://www.wikidata.org/wiki/Q1820283","display_name":"Learning vector quantization","level":3,"score":0.35010001063346863},{"id":"https://openalex.org/C2988494973","wikidata":"https://www.wikidata.org/wiki/Q179448","display_name":"Noise immunity","level":3,"score":0.3154999911785126},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.2888000011444092},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.2800000011920929},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.27869999408721924},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.2667999863624573},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.263700008392334},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2540000081062317},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.2502000033855438}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.11184","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.11184","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.11184","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.11184","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Mixture":[0],"of":[1,103,219,225,233],"Experts":[2],"(MoE)":[3],"models":[4],"have":[5],"achieved":[6],"great":[7],"success":[8],"by":[9,52,114],"significantly":[10],"improving":[11],"performance":[12,77],"while":[13],"maintaining":[14],"computational":[15],"efficiency":[16],"through":[17],"sparse":[18],"expert":[19,115],"activation.":[20],"However,":[21],"their":[22],"enormous":[23],"parameter":[24],"sizes":[25],"and":[26,107,169,174,189,244],"memory":[27],"demands":[28],"pose":[29],"major":[30],"challenges":[31],"for":[32,44,96,144,238],"deployment":[33,240],"in":[34,47,100,117,124],"resource-constrained":[35,246],"environments.":[36],"Vector":[37],"Quantization":[38],"(VQ)":[39],"offers":[40],"a":[41,54,135,156],"promising":[42],"approach":[43],"ultra-low-bit":[45],"compression":[46],"Large":[48],"Language":[49],"Models":[50],"(LLMs)":[51],"leveraging":[53],"codebook,":[55],"where":[56,155,179],"weight":[57,167],"vectors":[58],"are":[59,193],"mapped":[60],"to":[61,71,75,80,91,121,139,185,229],"the":[62,125,190,230],"most":[63],"similar":[64,94],"discrete":[65],"codewords.":[66],"Yet,":[67],"directly":[68],"applying":[69],"VQ":[70,90,137],"MoEs":[72],"often":[73],"leads":[74],"substantial":[76],"degradation":[78],"due":[79],"two":[81,149],"critical":[82],"obstacles:":[83],"(1)":[84,151],"redundant":[85],"representations":[86,95,188],"among":[87],"experts":[88],"cause":[89],"repeatedly":[92],"quantize":[93],"each":[97],"expert,":[98],"resulting":[99],"inefficient":[101],"use":[102],"limited":[104],"codebook":[105],"capacity;":[106],"(2)":[108,175],"cumulative":[109],"output":[110,177],"bias":[111],"is":[112,182],"amplified":[113],"aggregation":[116],"MoE":[118,202],"layers,":[119],"leading":[120],"distributional":[122],"shifts":[123],"quantized":[126,191],"outputs.":[127],"To":[128],"address":[129],"these":[130],"issues,":[131],"we":[132],"propose":[133],"KBVQ-MoE,":[134],"novel":[136],"framework":[138],"enhance":[140],"extremely":[141],"low-bit":[142],"quantization":[143,181,213,218],"MoE-based":[145],"LLMs.":[146],"KBVQ-MoE":[147,206],"integrates":[148],"techniques:":[150],"input-driven":[152],"redundancy":[153],"elimination,":[154],"Karhunen-Loeve":[157],"Transform":[158],"(KLT)":[159],"guided":[160],"singular":[161],"value":[162],"decomposition":[163],"(SVD)":[164],"extracts":[165],"dominant":[166],"components":[168],"shares":[170],"them":[171],"across":[172],"experts;":[173],"bias-corrected":[176],"stabilization,":[178],"vector":[180],"applied":[183],"only":[184],"expert-specific":[186],"(non-redundant)":[187],"outputs":[192],"corrected":[194],"via":[195],"channel-wise":[196],"affine":[197],"compensation.":[198],"Experiments":[199],"on":[200,241],"various":[201],"LLMs":[203],"demonstrate":[204],"that":[205],"preserves":[207],"accuracy":[208,224],"substantially":[209],"better":[210],"than":[211],"existing":[212],"methods.":[214],"For":[215],"example,":[216],"3-bit":[217],"Qwen1.5-MoE-A2.7B":[220],"achieves":[221],"an":[222],"average":[223],"67.99,":[226],"nearly":[227],"identical":[228],"FP16":[231],"baseline":[232],"68.07,":[234],"underscoring":[235],"KBVQ-MoE's":[236],"potential":[237],"efficient":[239],"edge":[242],"devices":[243],"other":[245],"platforms.":[247]},"counts_by_year":[],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2026-02-14T00:00:00"}