{"id":"https://openalex.org/W7162456234","doi":"https://doi.org/10.48550/arxiv.2605.24022","title":"Adaptive KV Cache Reuse for Fast Long-Context LLM Serving","display_name":"Adaptive KV Cache Reuse for Fast Long-Context LLM Serving","publication_year":2026,"publication_date":"2026-05-20","ids":{"openalex":"https://openalex.org/W7162456234","doi":"https://doi.org/10.48550/arxiv.2605.24022"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.24022","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.24022","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.24022","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137016602","display_name":"Fei li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"li, Fei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137029189","display_name":"Song Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Song","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137072785","display_name":"Yan Liu","orcid":"https://orcid.org/0000-0003-3197-9349"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072942012","display_name":"Jinhua Cui","orcid":"https://orcid.org/0000-0002-3252-7937"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cui, Jinhua","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016908677","display_name":"Shiqiang Nie","orcid":"https://orcid.org/0000-0003-2215-7159"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nie, Shiqiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137000877","display_name":"Jinyu Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jinyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137060259","display_name":"Weiguo Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Weiguo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.20319999754428864,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.20319999754428864,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.1737000048160553,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.16830000281333923,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.8213000297546387},{"id":"https://openalex.org/keywords/cache-algorithms","display_name":"Cache algorithms","score":0.612500011920929},{"id":"https://openalex.org/keywords/smart-cache","display_name":"Smart Cache","score":0.5985999703407288},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.5982000231742859},{"id":"https://openalex.org/keywords/reuse","display_name":"Reuse","score":0.5684999823570251},{"id":"https://openalex.org/keywords/cache-invalidation","display_name":"Cache invalidation","score":0.5605000257492065},{"id":"https://openalex.org/keywords/cache-pollution","display_name":"Cache pollution","score":0.44209998846054077},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.4244999885559082}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8457000255584717},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.8213000297546387},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.612500011920929},{"id":"https://openalex.org/C167713795","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"Smart Cache","level":5,"score":0.5985999703407288},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.5982000231742859},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.5684999823570251},{"id":"https://openalex.org/C25536678","wikidata":"https://www.wikidata.org/wiki/Q5015977","display_name":"Cache invalidation","level":5,"score":0.5605000257492065},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5321000218391418},{"id":"https://openalex.org/C113166858","wikidata":"https://www.wikidata.org/wiki/Q5015981","display_name":"Cache pollution","level":5,"score":0.44209998846054077},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.4244999885559082},{"id":"https://openalex.org/C201148951","wikidata":"https://www.wikidata.org/wiki/Q5015976","display_name":"Cache coloring","level":4,"score":0.4009000062942505},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.3610999882221222},{"id":"https://openalex.org/C193702766","wikidata":"https://www.wikidata.org/wiki/Q1414548","display_name":"Concurrency","level":2,"score":0.35910001397132874},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.35359999537467957},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.3513000011444092},{"id":"https://openalex.org/C51185590","wikidata":"https://www.wikidata.org/wiki/Q1017228","display_name":"Bus sniffing","level":5,"score":0.34290000796318054},{"id":"https://openalex.org/C120936851","wikidata":"https://www.wikidata.org/wiki/Q1408065","display_name":"MESI protocol","level":5,"score":0.3334999978542328},{"id":"https://openalex.org/C36340418","wikidata":"https://www.wikidata.org/wiki/Q7124288","display_name":"Page cache","level":5,"score":0.32440000772476196},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.3107999861240387},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.2867000102996826},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.27239999175071716},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.27059999108314514},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.2615000009536743},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.25270000100135803}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.24022","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.24022","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.24022","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.24022","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"In":[0],"long-context":[1,116,189],"Large":[2],"Language":[3],"Model":[4],"(LLM)":[5],"inference,":[6],"the":[7,13,18,55,123,145],"Time-To-First-Token":[8],"(TTFT)":[9],"latency":[10,155],"incurred":[11],"by":[12],"prefill":[14],"stage":[15],"has":[16],"become":[17],"foremost":[19],"bottleneck":[20],"limiting":[21],"interactive":[22],"performance":[23],"and":[24,60,100,109,134,169,177,188,198],"deployment":[25],"cost.":[26],"KV":[27,49,69,92,111,124,161],"Cache":[28,50,70,93,112],"reuse":[29,94,113],"offers":[30],"a":[31,86,107],"direct":[32],"path":[33],"to":[34,44,73,128,174,207,215],"reduce":[35],"redundant":[36],"prefill,":[37],"yet":[38],"traditional":[39],"prefix":[40],"caching":[41],"applies":[42],"only":[43,138],"strict-prefix":[45],"scenarios;":[46],"directly":[47],"reusing":[48,144],"in":[51,64],"non-prefix":[52,91],"settings":[53],"breaks":[54],"cross-chunk":[56],"global":[57],"attention":[58],"relationships":[59],"causes":[61],"significant":[62],"degradation":[63],"generation":[65,204],"quality.":[66],"When":[67],"reusable":[68],"is":[71],"offloaded":[72,214],"GPU-external":[74],"cache":[75,182],"pools,":[76],"I/O":[77],"overheads":[78],"across":[79,180],"heterogeneous":[80,181],"hardware":[81],"tiers":[82],"further":[83,158],"emerge":[84],"as":[85],"new":[87],"TTFT":[88,196,222],"bottleneck.":[89],"Efficient":[90],"therefore":[95],"requires":[96],"both":[97],"semantic-consistency":[98],"recovery":[99,130],"compute-I/O":[101],"co-optimization.":[102],"This":[103],"paper":[104],"presents":[105],"CacheTune,":[106],"frequency-guided":[108],"hardware-aware":[110,170],"system":[114],"for":[115],"LLM":[117],"serving.":[118],"CacheTune":[119,157,193,219],"first":[120],"identifies,":[121],"offline,":[122],"pairs":[125],"most":[126],"critical":[127],"cross-attention":[129],"through":[131,224],"frequency-domain":[132],"analysis,":[133],"then":[135],"selectively":[136],"recomputes":[137],"these":[139],"semantic-critical":[140],"tokens":[141],"online":[142],"while":[143,202],"remaining":[146],"KVs.":[147],"To":[148],"turn":[149],"this":[150],"semantic":[151],"selection":[152],"into":[153],"end-to-end":[154],"reduction,":[156],"combines":[159],"sparse":[160],"transfer,":[162],"multi-stream":[163],"asynchronous":[164],"overlap,":[165],"deferred":[166],"positional-encoding":[167],"recovery,":[168],"adaptive":[171,225],"recomputation-ratio":[172],"tuning":[173],"balance":[175],"computation":[176],"data":[178],"movement":[179],"pools.":[183],"Evaluations":[184],"on":[185],"mainstream":[186],"LLMs":[187],"tasks":[190],"show":[191],"that":[192],"achieves":[194],"3.72x-4.86x":[195],"speedup":[197,223],"3.93x-6.21x":[199],"higher":[200],"throughput":[201],"maintaining":[203],"quality":[205],"close":[206],"full":[208],"recompute.":[209],"Even":[210],"when":[211],"caches":[212],"are":[213],"I/O-bound":[216],"SSD/HDD":[217],"storage,":[218],"sustains":[220],"2.34x-2.36x":[221],"recomputation.":[226]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-27T00:00:00"}