{"id":"https://openalex.org/W7125908883","doi":"https://doi.org/10.48550/arxiv.2601.18999","title":"Randomization Boosts KV Caching, Learning Balances Query Load: A Joint Perspective","display_name":"Randomization Boosts KV Caching, Learning Balances Query Load: A Joint Perspective","publication_year":2026,"publication_date":"2026-01-26","ids":{"openalex":"https://openalex.org/W7125908883","doi":"https://doi.org/10.48550/arxiv.2601.18999"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2601.18999","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5124140709","display_name":"Fangzhou Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Fangzhou","raw_affiliation_strings":["Richard"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Richard","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124063821","display_name":"Sandeep Silwal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Silwal, Sandeep","raw_affiliation_strings":["Richard"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Richard","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124127061","display_name":"Qiuyi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiuyi","raw_affiliation_strings":["Richard"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Richard","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5124054358","display_name":"Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.07642266,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.7975000143051147,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.7975000143051147,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.03060000017285347,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.01889999955892563,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.7634999752044678},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5472000241279602},{"id":"https://openalex.org/keywords/reduction","display_name":"Reduction (mathematics)","score":0.46309998631477356},{"id":"https://openalex.org/keywords/eviction","display_name":"Eviction","score":0.4156000018119812},{"id":"https://openalex.org/keywords/cache-algorithms","display_name":"Cache algorithms","score":0.3873000144958496},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.35339999198913574},{"id":"https://openalex.org/keywords/cpu-cache","display_name":"CPU cache","score":0.352400004863739},{"id":"https://openalex.org/keywords/query-optimization","display_name":"Query optimization","score":0.35179999470710754},{"id":"https://openalex.org/keywords/online-algorithm","display_name":"Online algorithm","score":0.3474000096321106}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8026000261306763},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.7634999752044678},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5472000241279602},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.46309998631477356},{"id":"https://openalex.org/C2778227907","wikidata":"https://www.wikidata.org/wiki/Q1893186","display_name":"Eviction","level":2,"score":0.4156000018119812},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.3873000144958496},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.35339999198913574},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.352400004863739},{"id":"https://openalex.org/C157692150","wikidata":"https://www.wikidata.org/wiki/Q2919848","display_name":"Query optimization","level":2,"score":0.35179999470710754},{"id":"https://openalex.org/C196921405","wikidata":"https://www.wikidata.org/wiki/Q786431","display_name":"Online algorithm","level":2,"score":0.3474000096321106},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.3440999984741211},{"id":"https://openalex.org/C201148951","wikidata":"https://www.wikidata.org/wiki/Q5015976","display_name":"Cache coloring","level":4,"score":0.3398999869823456},{"id":"https://openalex.org/C25536678","wikidata":"https://www.wikidata.org/wiki/Q5015977","display_name":"Cache invalidation","level":5,"score":0.33009999990463257},{"id":"https://openalex.org/C192939062","wikidata":"https://www.wikidata.org/wiki/Q104840822","display_name":"Sargable","level":4,"score":0.3222000002861023},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.3174000084400177},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.3160000145435333},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3156000077724457},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.29820001125335693},{"id":"https://openalex.org/C174839445","wikidata":"https://www.wikidata.org/wiki/Q1134386","display_name":"Lock (firearm)","level":2,"score":0.29100000858306885},{"id":"https://openalex.org/C59687516","wikidata":"https://www.wikidata.org/wiki/Q5015938","display_name":"Cache-oblivious algorithm","level":5,"score":0.28929999470710754},{"id":"https://openalex.org/C113166858","wikidata":"https://www.wikidata.org/wiki/Q5015981","display_name":"Cache pollution","level":5,"score":0.28859999775886536},{"id":"https://openalex.org/C99016210","wikidata":"https://www.wikidata.org/wiki/Q5488129","display_name":"Query expansion","level":2,"score":0.2874000072479248},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.2856999933719635},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.28139999508857727},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.26600000262260437},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.26499998569488525},{"id":"https://openalex.org/C2781020372","wikidata":"https://www.wikidata.org/wiki/Q533093","display_name":"On the fly","level":2,"score":0.2630999982357025},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.262800008058548},{"id":"https://openalex.org/C192028432","wikidata":"https://www.wikidata.org/wiki/Q845739","display_name":"Query language","level":2,"score":0.257999986410141},{"id":"https://openalex.org/C102408133","wikidata":"https://www.wikidata.org/wiki/Q5156350","display_name":"Competitive analysis","level":3,"score":0.2540000081062317}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2601.18999","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2601.18999","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2601.18999","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2601.18999","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"KV":[0,84,109],"caching":[1],"is":[2,27,175],"a":[3],"fundamental":[4],"technique":[5],"for":[6],"accelerating":[7],"Large":[8],"Language":[9],"Model":[10],"(LLM)":[11],"inference":[12],"by":[13,135],"reusing":[14],"key-value":[15],"(KV)":[16],"pairs":[17],"from":[18],"previous":[19],"queries,":[20],"but":[21],"its":[22],"effectiveness":[23],"under":[24],"limited":[25],"memory":[26],"highly":[28],"sensitive":[29],"to":[30,101,115,149],"the":[31,73,80,93,170],"eviction":[32,40,86,111],"policy.":[33],"The":[34],"default":[35],"Least":[36],"Recently":[37],"Used":[38],"(LRU)":[39],"algorithm":[41],"struggles":[42],"with":[43,112,119],"dynamic":[44],"online":[45],"query":[46,55,88,124],"arrivals,":[47],"especially":[48],"in":[49,151,157,161,167],"multi-LLM":[50],"serving":[51],"scenarios,":[52],"where":[53],"balancing":[54,123],"load":[56,125],"across":[57,138],"workers":[58],"and":[59,87,99,126,141,164],"maximizing":[60],"cache":[61,85,110,127,152],"hit":[62,128,153],"rate":[63],"of":[64,96,147],"each":[65],"worker":[66],"are":[67,133],"inherently":[68],"conflicting":[69],"objectives.":[70],"We":[71],"give":[72],"first":[74],"unified":[75],"mathematical":[76],"model":[77],"that":[78,104],"captures":[79],"core":[81],"trade-offs":[82],"between":[83],"routing.":[89],"Our":[90,130,173],"analysis":[91],"reveals":[92],"theoretical":[94,131],"limitations":[95],"existing":[97],"methods":[98,114],"leads":[100],"principled":[102],"algorithms":[103],"integrate":[105],"provably":[106],"competitive":[107],"randomized":[108],"learning-based":[113],"adaptively":[116],"route":[117],"queries":[118],"evolving":[120],"patterns,":[121],"thus":[122],"rate.":[129],"results":[132],"validated":[134],"extensive":[136],"experiments":[137],"4":[139],"benchmarks":[140],"3":[142],"prefix-sharing":[143],"settings,":[144],"demonstrating":[145],"improvements":[146],"up":[148],"6.92$\\times$":[150],"rate,":[154],"11.96$\\times$":[155],"reduction":[156,160],"latency,":[158],"14.06$\\times$":[159],"time-to-first-token":[162],"(TTFT),":[163],"77.4%":[165],"increase":[166],"throughput":[168],"over":[169],"state-of-the-art":[171],"methods.":[172],"code":[174],"available":[176],"at":[177],"https://github.com/fzwark/KVRouting.":[178]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-01-29T00:00:00"}