{"id":"https://openalex.org/W7161788698","doi":"https://doi.org/10.48550/arxiv.2605.18825","title":"Not All Tokens Are Worth Caching: Learning Semantic-Aware Eviction for LLM Prefix Caches","display_name":"Not All Tokens Are Worth Caching: Learning Semantic-Aware Eviction for LLM Prefix Caches","publication_year":2026,"publication_date":"2026-05-12","ids":{"openalex":"https://openalex.org/W7161788698","doi":"https://doi.org/10.48550/arxiv.2605.18825"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.18825","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.18825","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.18825","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054602800","display_name":"Shaoke Fang","orcid":"https://orcid.org/0009-0001-9757-1031"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fang, Shaoke","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136546735","display_name":"Ziang Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Ziang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136595832","display_name":"Wenfei Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Wenfei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136587106","display_name":"Jiatong Ji","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ji, Jiatong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136559349","display_name":"Qingsong Liu","orcid":"https://orcid.org/0000-0002-4137-9163"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Qingsong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136507344","display_name":"Ruizhi Pu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pu, Ruizhi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.28439998626708984,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.28439998626708984,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.24400000274181366,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.11959999799728394,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.7407000064849854},{"id":"https://openalex.org/keywords/prefix","display_name":"Prefix","score":0.6833999752998352},{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.6161999702453613},{"id":"https://openalex.org/keywords/reuse","display_name":"Reuse","score":0.42730000615119934},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.3614000082015991},{"id":"https://openalex.org/keywords/queue","display_name":"Queue","score":0.3549000024795532},{"id":"https://openalex.org/keywords/cpu-cache","display_name":"CPU cache","score":0.3540000021457672}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8306000232696533},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.7407000064849854},{"id":"https://openalex.org/C141603448","wikidata":"https://www.wikidata.org/wiki/Q134830","display_name":"Prefix","level":2,"score":0.6833999752998352},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.6161999702453613},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.5232999920845032},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.428600013256073},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.42730000615119934},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.3614000082015991},{"id":"https://openalex.org/C160403385","wikidata":"https://www.wikidata.org/wiki/Q220543","display_name":"Queue","level":2,"score":0.3549000024795532},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.3540000021457672},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.33219999074935913},{"id":"https://openalex.org/C2778227907","wikidata":"https://www.wikidata.org/wiki/Q1893186","display_name":"Eviction","level":2,"score":0.3231000006198883},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.32100000977516174},{"id":"https://openalex.org/C7366592","wikidata":"https://www.wikidata.org/wiki/Q1255620","display_name":"Dram","level":2,"score":0.30390000343322754},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.301800012588501},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.30149999260902405},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.27250000834465027},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.2685999870300293},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.2549000084400177}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.18825","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.18825","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.18825","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.18825","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Prefix":[0],"caching":[1],"is":[2,40],"a":[3,56,76,117,131,160,180,243],"key":[4],"optimization":[5],"in":[6,95,149,155],"Large":[7],"Language":[8],"Model":[9],"(LLM)":[10],"serving,":[11],"reusing":[12],"attention":[13],"Key-Value":[14],"(KV)":[15],"states":[16],"across":[17,216],"requests":[18,151],"with":[19,141],"shared":[20],"prompt":[21],"prefixes":[22],"to":[23,92,138,209,237],"reduce":[24],"expensive":[25],"prefill":[26],"computation.":[27],"However,":[28],"its":[29],"benefit":[30],"depends":[31],"critically":[32],"on":[33],"the":[34,167],"eviction":[35,101,121,176],"policy":[36,102,122],"as":[37,46],"GPU":[38],"memory":[39],"scarce,":[41],"and":[42,87,152,178,199,205],"existing":[43,100],"policies":[44],"such":[45],"LRU":[47],"largely":[48],"treat":[49],"cached":[50],"blocks":[51,137],"uniformly.":[52],"This":[53],"view":[54],"ignores":[55],"fundamental":[57],"property":[58],"of":[59,170],"LLM":[60],"prompts:":[61],"not":[62],"all":[63,187],"tokens":[64],"are":[65],"equally":[66],"worth":[67],"caching.":[68],"We":[69],"show":[70],"that":[71,123,134,165,221],"different":[72,171],"token":[73,162,172],"types":[74,173],"within":[75],"prompt,":[77],"including":[78,190],"system":[79],"prompts,":[80],"user":[81],"queries,":[82],"tool":[83],"outputs,":[84],"model":[85],"responses,":[86],"chain-of-thought":[88],"reasoning,":[89],"exhibit":[90],"up":[91,236],"756x":[93],"variation":[94],"reuse":[96,148,154,168],"rates,":[97],"yet":[98],"no":[99],"exploits":[103],"this":[104,107,125],"signal.":[105],"In":[106],"paper,":[108],"we":[109,219],"present":[110],"SAECache":[111,222],"(Semantic-Adaptive":[112],"Eviction":[113],"for":[114,186],"prefix":[115,119],"caches),":[116],"semantic-adaptive":[118],"cache":[120],"addresses":[124],"gap":[126],"through":[127,175],"three":[128],"innovations:":[129],"(1)":[130],"multi-queue":[132],"architecture":[133],"routes":[135],"KV":[136],"task-specific":[139],"queues":[140],"tailored":[142],"priority":[143],"metrics,":[144],"capturing":[145],"both":[146],"session":[147],"multi-turn":[150],"structural":[153],"templated":[156],"single-turn":[157],"requests;":[158],"(2)":[159],"semantic-aware":[161],"weighting":[163],"mechanism":[164],"learns":[166],"value":[169],"online":[174,183],"feedback;":[177],"(3)":[179],"fully":[181],"adaptive":[182,247],"learning":[184],"schema":[185],"parameter":[188],"updates,":[189],"log-normal":[191],"timing":[192],"parameters,":[193],"position":[194],"decay":[195],"power,":[196],"queue":[197],"weights,":[198],"meta-parameters,":[200],"which":[201],"eliminates":[202],"manual":[203],"tuning":[204],"enables":[206],"automatic":[207],"adaptation":[208],"deployment-specific":[210],"workload":[211,240],"characteristics.":[212],"Through":[213],"extensive":[214],"evaluation":[215],"heterogeneous":[217],"workloads,":[218],"demonstrate":[220],"achieves":[223],"1.4x-2.7x":[224],"TTFT":[225],"improvement":[226],"over":[227],"production-style":[228],"baselines,":[229],"while":[230],"fixed-parameter":[231],"alternatives":[232],"can":[233],"degrade":[234],"by":[235],"2.7x":[238],"under":[239],"mismatch":[241],"--":[242],"failure":[244],"mode":[245],"our":[246],"approach":[248],"avoids":[249],"entirely.":[250]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-21T00:00:00"}