{"id":"https://openalex.org/W7160908419","doi":"https://doi.org/10.48550/arxiv.2605.09649","title":"Make Each Token Count: Towards Improving Long-Context Performance with KV Cache Eviction","display_name":"Make Each Token Count: Towards Improving Long-Context Performance with KV Cache Eviction","publication_year":2026,"publication_date":"2026-05-10","ids":{"openalex":"https://openalex.org/W7160908419","doi":"https://doi.org/10.48550/arxiv.2605.09649"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.09649","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09649","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.09649","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135983626","display_name":"Ngoc Bui","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bui, Ngoc","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135965526","display_name":"Hieu Trung Nguyen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nguyen, Hieu Trung","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135936887","display_name":"Arman Cohan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cohan, Arman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135978538","display_name":"Rex Ying","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ying, Rex","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.32179999351501465,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.32179999351501465,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.22840000689029694,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.09399999678134918,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.7390000224113464},{"id":"https://openalex.org/keywords/eviction","display_name":"Eviction","score":0.6879000067710876},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.5891000032424927},{"id":"https://openalex.org/keywords/translation-lookaside-buffer","display_name":"Translation lookaside buffer","score":0.5702000260353088},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.4934999942779541},{"id":"https://openalex.org/keywords/cpu-cache","display_name":"CPU cache","score":0.4496000111103058},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.39250001311302185}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7671999931335449},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.7390000224113464},{"id":"https://openalex.org/C2778227907","wikidata":"https://www.wikidata.org/wiki/Q1893186","display_name":"Eviction","level":2,"score":0.6879000067710876},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.5891000032424927},{"id":"https://openalex.org/C116007543","wikidata":"https://www.wikidata.org/wiki/Q1071403","display_name":"Translation lookaside buffer","level":4,"score":0.5702000260353088},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.5062999725341797},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.4934999942779541},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.4496000111103058},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.39250001311302185},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3862999975681305},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3476000130176544},{"id":"https://openalex.org/C136085584","wikidata":"https://www.wikidata.org/wiki/Q910289","display_name":"Overlay","level":2,"score":0.3287999927997589},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.3147999942302704},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3107999861240387},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.31060001254081726},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.29649999737739563},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.2939999997615814},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.28439998626708984},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.26820001006126404}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.09649","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09649","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.09649","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09649","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"key-value":[1],"(KV)":[2],"cache":[3,133],"is":[4,37,41,193],"a":[5,73,86,101,116,155,196,201],"major":[6],"bottleneck":[7],"in":[8,45,121],"long-context":[9,163,205],"inference,":[10],"where":[11],"memory":[12,88,177],"and":[13,100,112,128,149,165,168],"computation":[14],"grow":[15],"with":[16],"sequence":[17],"length.":[18],"Existing":[19],"KV":[20,76,98,176,191],"eviction":[21,60,77,119,192],"methods":[22],"reduce":[23],"this":[24],"cost":[25],"but":[26,199],"typically":[27],"degrade":[28],"performance":[29],"relative":[30],"to":[31,96],"full-cache":[32,39,182],"inference.":[33,183],"Our":[34],"key":[35],"insight":[36],"that":[38,79,141,187],"attention":[40,52,147],"not":[42,194],"always":[43],"optimal:":[44],"long":[46],"contexts,":[47],"irrelevant":[48],"tokens":[49,123,145],"can":[50,61],"dilute":[51],"away":[53],"from":[54,124],"useful":[55,144],"evidence,":[56],"so":[57],"selective,":[58],"learnable":[59],"improve":[62],"generation":[63],"rather":[64],"than":[65],"merely":[66],"approximate":[67],"the":[68],"full":[69],"cache.":[70],"We":[71,135],"introduce":[72],"global":[74,118],"retention-based":[75],"method":[78,173],"learns":[80],"each":[81],"token's":[82],"future":[83,159],"utility":[84,94],"under":[85],"unified":[87],"budget.":[89],"Lightweight":[90],"retention":[91,153],"gates":[92],"assign":[93],"scores":[95,108],"cached":[97],"entries,":[99],"shared":[102],"final":[103],"scoring":[104],"projection":[105],"calibrates":[106],"these":[107],"across":[109],"all":[110],"layers":[111],"heads.":[113],"This":[114],"enables":[115],"single":[117],"policy":[120],"which":[122],"different":[125],"layers,":[126],"heads,":[127],"modalities":[129],"compete":[130],"directly":[131],"for":[132,158,203],"capacity.":[134],"further":[136],"provide":[137],"theoretical":[138],"analysis":[139],"showing":[140],"preferentially":[142],"retaining":[143],"reduces":[146,175],"dilution,":[148],"we":[150],"justify":[151],"geometric":[152],"as":[154],"query-agnostic":[156],"proxy":[157],"utility.":[160],"Across":[161],"diverse":[162],"language":[164],"vision-language":[166],"reasoning,":[167],"multi-turn":[169],"dialogue":[170],"benchmarks,":[171],"our":[172],"substantially":[174],"while":[178],"matching":[179],"or":[180],"surpassing":[181],"These":[184],"results":[185],"suggest":[186],"learned,":[188],"globally":[189],"calibrated":[190],"only":[195],"compression":[197],"technique,":[198],"also":[200],"mechanism":[202],"improving":[204],"reasoning.":[206]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-13T00:00:00"}