{"id":"https://openalex.org/W7204930987","doi":"https://doi.org/10.48550/arxiv.2608.28911","title":"SemKV: Semantic Mixed-Precision KV Cache Quantization Guided by the Quality Cliff for Long-Context LLM Inference","display_name":"SemKV: Semantic Mixed-Precision KV Cache Quantization Guided by the Quality Cliff for Long-Context LLM Inference","publication_year":2026,"publication_date":"2026-08-28","ids":{"openalex":"https://openalex.org/W7204930987","doi":"https://doi.org/10.48550/arxiv.2608.28911"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.28911","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.28911","pdf_url":"https://arxiv.org/pdf/2608.28911","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.28911","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5148870503","display_name":"Daeha Lee","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I142401562","display_name":"Electronics and Telecommunications Research Institute","ror":"https://ror.org/03ysstz10","country_code":"KR","type":"facility","lineage":["https://openalex.org/I142401562","https://openalex.org/I2801339556","https://openalex.org/I4210144908","https://openalex.org/I4387152098"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Lee, Daeha","raw_affiliation_strings":["Electronics and Telecommunications Research Institute (ETRI) , Daejeon , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Electronics and Telecommunications Research Institute (ETRI) , Daejeon , Republic of Korea","institution_ids":["https://openalex.org/I142401562"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148836515","display_name":"Do-Hyung Kim","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I142401562","display_name":"Electronics and Telecommunications Research Institute","ror":"https://ror.org/03ysstz10","country_code":"KR","type":"facility","lineage":["https://openalex.org/I142401562","https://openalex.org/I2801339556","https://openalex.org/I4210144908","https://openalex.org/I4387152098"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Kim, Do-Hyung","raw_affiliation_strings":["Electronics and Telecommunications Research Institute (ETRI) , Daejeon , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Electronics and Telecommunications Research Institute (ETRI) , Daejeon , Republic of Korea","institution_ids":["https://openalex.org/I142401562"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101480473","display_name":"Jaehong Kim","orcid":"https://orcid.org/0000-0002-9111-9976","observed_orcids":["https://orcid.org/0000-0002-9111-9976"]},"institutions":[{"id":"https://openalex.org/I142401562","display_name":"Electronics and Telecommunications Research Institute","ror":"https://ror.org/03ysstz10","country_code":"KR","type":"facility","lineage":["https://openalex.org/I142401562","https://openalex.org/I2801339556","https://openalex.org/I4210144908","https://openalex.org/I4387152098"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Kim, Jae-Hong","raw_affiliation_strings":["Electronics and Telecommunications Research Institute (ETRI) , Daejeon , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Electronics and Telecommunications Research Institute (ETRI) , Daejeon , Republic of Korea","institution_ids":["https://openalex.org/I142401562"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I142401562"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.8809000253677368,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.8809000253677368,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.025299999862909317,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10363","display_name":"Low-power high-performance VLSI design","score":0.01119999960064888,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mixed-precision-quantization","display_name":"mixed-precision quantization","score":0.8320000171661377},{"id":"https://openalex.org/keywords/long-context-llm-inference","display_name":"long-context LLM inference","score":0.8629999756813049},{"id":"https://openalex.org/keywords/memory-bottleneck","display_name":"memory bottleneck","score":0.6460000276565552},{"id":"https://openalex.org/keywords/token-pruning","display_name":"token pruning","score":0.6000000238418579},{"id":"https://openalex.org/keywords/llama-3-1","display_name":"Llama 3.1","score":0.6690000295639038},{"id":"https://openalex.org/keywords/mistral-7b","display_name":"Mistral-7B","score":0.9629999995231628}],"concepts":[{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.7046999931335449},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6255000233650208},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.5123999714851379},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5031999945640564},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.45969998836517334},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4052000045776367},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4020000100135803},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.3865000009536743},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.38190001249313354},{"id":"https://openalex.org/C92757383","wikidata":"https://www.wikidata.org/wiki/Q382497","display_name":"Affine transformation","level":2,"score":0.3716000020503998},{"id":"https://openalex.org/C187691185","wikidata":"https://www.wikidata.org/wiki/Q2020720","display_name":"Grid","level":2,"score":0.3698999881744385},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3610000014305115},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33340001106262207},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.3253999948501587},{"id":"https://openalex.org/C199833920","wikidata":"https://www.wikidata.org/wiki/Q612536","display_name":"Vector quantization","level":2,"score":0.29490000009536743},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.28049999475479126},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.26840001344680786},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.2671999931335449},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.2639000117778778},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.26100000739097595},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.25949999690055847},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.25209999084472656},{"id":"https://openalex.org/C44648626","wikidata":"https://www.wikidata.org/wiki/Q1049848","display_name":"Percentage point","level":2,"score":0.25060001015663147}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.28911","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.28911","pdf_url":"https://arxiv.org/pdf/2608.28911","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.28911","doi":"https://doi.org/10.48550/arxiv.2608.28911","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.28911","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.28911","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.28911","pdf_url":"https://arxiv.org/pdf/2608.28911","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320328359","display_name":"Ministry of Science and ICT, South Korea","ror":"https://ror.org/01wpjm123"},{"id":"https://openalex.org/F4320335489","display_name":"Institute for Information and Communications Technology Promotion","ror":"https://ror.org/01g0hqq23"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204930987.pdf","grobid_xml":"https://content.openalex.org/works/W7204930987.grobid-xml"},"referenced_works_count":16,"referenced_works":["https://openalex.org/W1598866093","https://openalex.org/W2940744433","https://openalex.org/W2973727699","https://openalex.org/W2988394319","https://openalex.org/W3015468748","https://openalex.org/W4322718191","https://openalex.org/W4327810158","https://openalex.org/W4384918448","https://openalex.org/W4387561528","https://openalex.org/W4392018977","https://openalex.org/W4392363977","https://openalex.org/W4392678657","https://openalex.org/W4414359395","https://openalex.org/W4416034896","https://openalex.org/W4416915242","https://openalex.org/W7131102605"],"related_works":["https://openalex.org/W4417253745","https://openalex.org/W4407308523","https://openalex.org/W4415799217","https://openalex.org/W7203613154","https://openalex.org/W7163635936","https://openalex.org/W4410582506","https://openalex.org/W4392363977","https://openalex.org/W7125967601","https://openalex.org/W7118619038","https://openalex.org/W4416034896"],"abstract_inverted_index":{"The":[0,81,179],"key-value":[1],"(KV)":[2],"cache":[3],"is":[4,46,101],"the":[5,97,157,166,173,182,185],"dominant":[6],"memory":[7,154],"bottleneck":[8],"of":[9,99],"long-context":[10],"large":[11],"language":[12],"model":[13],"(LLM)":[14],"inference,":[15],"growing":[16],"linearly":[17],"with":[18,42,133,160],"context":[19],"length.":[20],"We":[21],"show":[22],"that":[23,69],"uniform":[24,107],"KV":[25,51,141],"quantization":[26,73,108],"on":[27],"a":[28,36,63,118,128,151,161],"fractional-bit":[29],"grid":[30,102],"does":[31],"not":[32],"degrade":[33],"gracefully:":[34],"under":[35],"prespecified":[37],"multi-seed":[38],"statistical":[39],"protocol,":[40],"Llama-3.1-8B-Instruct":[41],"an":[43],"affine":[44,158],"quantizer":[45,163],"statistically":[47,94,135],"indistinguishable":[48],"from":[49,139],"FP16":[50,147],"down":[52],"to":[53,79,177],"2.322":[54],"code":[55],"bits/value":[56],"and":[57,74,77,121,145],"collapses":[58],"at":[59],"2.0":[60],"bits":[61],"-":[62],"quality":[64,137],"cliff":[65,82,167,183],"in":[66,71,168],"(2.0,":[67],"2.322]":[68],"reappears":[70],"generation-time":[72],"multi-turn":[75],"dialogue":[76],"transfers":[78],"Mistral-7B.":[80],"reframes":[83],"importance-aware":[84],"mixed":[85],"precision:":[86],"above":[87,191],"it,":[88],"eight":[89],"model-internal":[90,119],"importance":[91],"indicators":[92],"are":[93],"interchangeable,":[95],"so":[96],"benefit":[98],"mixing":[100],"interpolation,":[103],"reaching":[104],"average":[105],"precisions":[106],"cannot":[109],"realize.":[110],"SemKV":[111],"preserves":[112],"every":[113,169],"token,":[114],"ranks":[115],"tokens":[116],"by":[117],"score,":[120],"assigns":[122],"two":[123],"adjacent":[124],"above-cliff":[125],"precisions,":[126],"achieving":[127],"measured":[129],"6.0x":[130],"storage":[131],"reduction":[132],"no":[134],"detectable":[136],"difference":[138],"full":[140],"(n=900,":[142],"three":[143],"seeds),":[144],"outperforming":[146],"token":[148],"pruning":[149],"granted":[150],"1.5x":[152],"larger":[153],"budget.":[155],"Replacing":[156],"base":[159],"distortion-optimized":[162],"(TurboQuant-MSE)":[164],"lowers":[165],"protocol":[170],"tested,":[171],"raising":[172],"no-detectable-loss":[174],"operating":[175],"point":[176],"7.9x.":[178],"recipe:":[180],"measure":[181],"for":[184],"target":[186],"deployment":[187],"setting,":[188],"then":[189],"interpolate":[190],"it.":[192]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-02T00:00:00"}