{"id":"https://openalex.org/W7223068331","doi":"https://doi.org/10.48550/arxiv.2610.24179","title":"KVCompass: A Discovery Plane for Internet-Scale Prefix Cache Sharing in LLM Serving","display_name":"KVCompass: A Discovery Plane for Internet-Scale Prefix Cache Sharing in LLM Serving","publication_year":2026,"publication_date":"2026-10-16","ids":{"openalex":"https://openalex.org/W7223068331","doi":"https://doi.org/10.48550/arxiv.2610.24179"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.24179","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.24179","pdf_url":"https://arxiv.org/pdf/2610.24179","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.24179","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139179988","display_name":"Shiyi Ling","orcid":"https://orcid.org/0009-0007-8914-114X","observed_orcids":["https://orcid.org/0009-0007-8914-114X"]},"institutions":[{"id":"https://openalex.org/I185103710","display_name":"University of California, Santa Cruz","ror":"https://ror.org/03s65by71","country_code":"US","type":"education","lineage":["https://openalex.org/I185103710"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ling, Shiyi","raw_affiliation_strings":["University of California , Santa Cruz , USA FEI FANG ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Cruz , USA FEI FANG ,","institution_ids":["https://openalex.org/I185103710"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5159138637","display_name":"Fei Fang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I185103710","display_name":"University of California, Santa Cruz","ror":"https://ror.org/03s65by71","country_code":"US","type":"education","lineage":["https://openalex.org/I185103710"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fang, Fei","raw_affiliation_strings":["University of California , Santa Cruz , Santa Cruz , California , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Cruz , Santa Cruz , California , USA","institution_ids":["https://openalex.org/I185103710"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100737734","display_name":"Chen Qian","orcid":"https://orcid.org/0000-0002-6882-9590","observed_orcids":["https://orcid.org/0000-0002-6882-9590","https://orcid.org/0000-0001-6498-4262"]},"institutions":[{"id":"https://openalex.org/I185103710","display_name":"University of California, Santa Cruz","ror":"https://ror.org/03s65by71","country_code":"US","type":"education","lineage":["https://openalex.org/I185103710"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qian, Chen","raw_affiliation_strings":["University of California , Santa Cruz , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Cruz , USA","institution_ids":["https://openalex.org/I185103710"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I185103710"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.48559999465942383,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.48559999465942383,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.2094999998807907,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.05209999904036522,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-model-serving","display_name":"large language model serving","score":0.7670000195503235},{"id":"https://openalex.org/keywords/time-to-first-token","display_name":"time-to-first-token","score":0.8410000205039978},{"id":"https://openalex.org/keywords/kv-cache-eviction","display_name":"KV cache eviction","score":0.5059999823570251},{"id":"https://openalex.org/keywords/edge-caching","display_name":"edge caching","score":0.33399999141693115},{"id":"https://openalex.org/keywords/load-balancing","display_name":"load balancing","score":0.625}],"concepts":[{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.8741000294685364},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8356999754905701},{"id":"https://openalex.org/C25536678","wikidata":"https://www.wikidata.org/wiki/Q5015977","display_name":"Cache invalidation","level":5,"score":0.6212000250816345},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.5835000276565552},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.5388000011444092},{"id":"https://openalex.org/C167713795","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"Smart Cache","level":5,"score":0.49900001287460327},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.4528999924659729},{"id":"https://openalex.org/C201148951","wikidata":"https://www.wikidata.org/wiki/Q5015976","display_name":"Cache coloring","level":4,"score":0.4147000014781952},{"id":"https://openalex.org/C113166858","wikidata":"https://www.wikidata.org/wiki/Q5015981","display_name":"Cache pollution","level":5,"score":0.39890000224113464},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.3772999942302704},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.37720000743865967},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3481000065803528},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.33410000801086426},{"id":"https://openalex.org/C36340418","wikidata":"https://www.wikidata.org/wiki/Q7124288","display_name":"Page cache","level":5,"score":0.32919999957084656},{"id":"https://openalex.org/C51185590","wikidata":"https://www.wikidata.org/wiki/Q1017228","display_name":"Bus sniffing","level":5,"score":0.30720001459121704},{"id":"https://openalex.org/C141603448","wikidata":"https://www.wikidata.org/wiki/Q134830","display_name":"Prefix","level":2,"score":0.3012000024318695},{"id":"https://openalex.org/C190290938","wikidata":"https://www.wikidata.org/wiki/Q387015","display_name":"Trie","level":3,"score":0.2930000126361847},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.25999999046325684},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.25870001316070557},{"id":"https://openalex.org/C59687516","wikidata":"https://www.wikidata.org/wiki/Q5015938","display_name":"Cache-oblivious algorithm","level":5,"score":0.25769999623298645}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.24179","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.24179","pdf_url":"https://arxiv.org/pdf/2610.24179","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.24179","doi":"https://doi.org/10.48550/arxiv.2610.24179","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.24179","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.24179","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.24179","pdf_url":"https://arxiv.org/pdf/2610.24179","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2774216771","display_name":"NeTS: SMALL: Develop Core Techniques and Applications of Learned and Disaggregated In-Network Lookups","funder_award_id":"2420632","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3626882726","display_name":"CNS Core: Small: Core Scheduling Techniques and Programming Abstractions for Scalable Serverless Edge Computing Engine","funder_award_id":"2322919","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4052096878","display_name":"Collaborative Research: NeTS: Small: Resource-efficient Consolidated Data Plane for Network Functions","funder_award_id":"2426031","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G985058349","display_name":"ExpandQISE: Track 1: Education and Research of System and Network Supports for Quantum Cloud","funder_award_id":"2426940","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7223068331.pdf","grobid_xml":"https://content.openalex.org/works/W7223068331.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W2073965851","https://openalex.org/W2116807588","https://openalex.org/W2163059190","https://openalex.org/W2964315738","https://openalex.org/W4253233651","https://openalex.org/W4390690436","https://openalex.org/W4400104234","https://openalex.org/W4401211704","https://openalex.org/W4401306886","https://openalex.org/W4402671258","https://openalex.org/W4403752360","https://openalex.org/W4404401017","https://openalex.org/W4415284287","https://openalex.org/W4416011790","https://openalex.org/W4417150458","https://openalex.org/W7128790072","https://openalex.org/W7140879077","https://openalex.org/W7154723983","https://openalex.org/W7155497146"],"related_works":["https://openalex.org/W7172427218","https://openalex.org/W7212604482","https://openalex.org/W4413360469","https://openalex.org/W7223020526","https://openalex.org/W7169827217","https://openalex.org/W7172414975","https://openalex.org/W4417150458","https://openalex.org/W7162298914","https://openalex.org/W4415003901","https://openalex.org/W7162456234"],"abstract_inverted_index":{"KV":[0,65,75,103,122,125,132,145,175],"cache":[1,66,76,123,126,133,146,176],"reuse":[2,54,177],"is":[3,79,96],"one":[4],"of":[5,100,226],"the":[6,30,42,108,161,170,205,217,227],"most":[7],"effective":[8],"ways":[9],"to":[10,22,166,169,208,215],"reduce":[11],"LLM":[12],"inference":[13],"cost,":[14],"but":[15],"today":[16],"its":[17,101],"benefits":[18],"are":[19,105],"largely":[20],"confined":[21],"a":[23,27,63,97,140,185,190,220],"single":[24],"cluster.":[25],"When":[26],"provider":[28],"serves":[29],"same":[31,43],"model":[32],"across":[33,55],"geographically":[34],"distributed":[35],"regions,":[36],"each":[37,68,164],"region":[38],"may":[39],"repeatedly":[40],"prefill":[41],"popular":[44],"prefixes,":[45],"wasting":[46],"GPU":[47],"time":[48],"and":[49,113,121,180,189,219],"increasing":[50],"time-to-first-token":[51],"(TTFT).":[52],"Exploiting":[53],"regions":[56],"involves":[57],"discovering":[58],"which":[59],"worker":[60,171],"currently":[61],"holds":[62],"reusable":[64],"for":[67,130,143],"request's":[69],"prefix.":[70],"We":[71,137],"show":[72],"that":[73,110,172,197],"this":[74],"discovery":[77,127,141,149],"problem":[78],"structurally":[80],"harder":[81],"than":[82,204],"conventional":[83],"object":[84],"lookup.":[85],"Unlike":[86],"hash-placement":[87],"mechanisms":[88],"such":[89],"as":[90],"DHTs,":[91],"where":[92],"an":[93],"object's":[94],"location":[95],"deterministic":[98],"function":[99],"key,":[102],"caches":[104],"created":[106],"on":[107],"workers":[109],"serve":[111],"requests,":[112],"their":[114],"holder":[115],"sets":[116],"change":[117],"with":[118,155],"subsequent":[119],"requests":[120],"evictions.":[124],"therefore":[128],"calls":[129],"decoupling":[131],"identity":[134],"from":[135],"placement.":[136],"present":[138],"KVCompass,":[139],"plane":[142],"Internet-scale":[144],"sharing.":[147],"The":[148],"index":[150],"supports":[151],"variable-length":[152],"prefix":[153],"lookup":[154],"constant-time":[156],"local":[157],"table":[158],"reads":[159],"at":[160],"edge,":[162],"allowing":[163],"request":[165],"be":[167],"forwarded":[168],"best":[173],"balances":[174],"against":[178],"load":[179,222],"network":[181],"distance.":[182],"Evaluation":[183,195],"includes":[184],"64-worker":[186],"geo-distributed":[187],"simulation":[188],"14-worker":[191],"three-continent":[192],"real-machine":[193],"testbed.":[194],"shows":[196],"KVCompass":[198],"achieves":[199],"$1.3-14\\times$":[200],"lower":[201,210],"P99":[202,211],"TTFT":[203],"baselines,":[206],"up":[207,214],"$3.7\\times$":[209],"end-to-end":[212],"latency,":[213],"$4.7\\times$":[216],"throughput,":[218],"peak":[221],"factor":[223],"within":[224],"2%":[225],"Oracle":[228],"floor.":[229]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-21T00:00:00"}