{"id":"https://openalex.org/W7082968261","doi":"https://doi.org/10.1145/3749168","title":"HotPrefix: Hotness-Aware KV Cache Scheduling for Efficient Prefix Sharing in LLM Inference Systems","display_name":"HotPrefix: Hotness-Aware KV Cache Scheduling for Efficient Prefix Sharing in LLM Inference Systems","publication_year":2025,"publication_date":"2025-09-22","ids":{"openalex":"https://openalex.org/W7082968261","doi":"https://doi.org/10.1145/3749168"},"language":"en","primary_location":{"id":"doi:10.1145/3749168","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3749168","pdf_url":null,"source":{"id":"https://openalex.org/S4387289859","display_name":"Proceedings of the ACM on Management of Data","issn_l":"2836-6573","issn":["2836-6573"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Management of Data","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Yuhang Li","orcid":"https://orcid.org/0009-0003-1994-8133"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuhang Li","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, Jiangsu, China"],"raw_orcid":"https://orcid.org/0009-0003-1994-8133","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, Jiangsu, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Rong Gu","orcid":"https://orcid.org/0000-0002-1565-9997"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rong Gu","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, Jiangsu, China"],"raw_orcid":"https://orcid.org/0000-0002-1565-9997","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, Jiangsu, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Chengying Huan","orcid":"https://orcid.org/0000-0002-3154-3580"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chengying Huan","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, Jiangsu, China"],"raw_orcid":"https://orcid.org/0000-0002-3154-3580","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, Jiangsu, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhibin Wang","orcid":"https://orcid.org/0000-0001-9204-4075"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhibin Wang","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, Jiangsu, China"],"raw_orcid":"https://orcid.org/0000-0001-9204-4075","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, Jiangsu, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Renjie Yao","orcid":"https://orcid.org/0009-0000-9556-5773"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Renjie Yao","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, Jiangsu, China"],"raw_orcid":"https://orcid.org/0009-0000-9556-5773","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, Jiangsu, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Chen Tian","orcid":"https://orcid.org/0000-0003-2710-7628"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chen Tian","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, Jiangsu, China"],"raw_orcid":"https://orcid.org/0000-0003-2710-7628","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, Jiangsu, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"last","author":{"id":null,"display_name":"Guihai Chen","orcid":"https://orcid.org/0000-0002-6934-1685"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guihai Chen","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, Jiangsu, China"],"raw_orcid":"https://orcid.org/0000-0002-6934-1685","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, Jiangsu, China","institution_ids":["https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I881766915"],"apc_list":null,"apc_paid":null,"fwci":6.6032,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.96748949,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"3","issue":"4","first_page":"1","last_page":"27"},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T12157","display_name":"Geochemistry and Geologic Mapping","score":0.7087000012397766,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12157","display_name":"Geochemistry and Geologic Mapping","score":0.7087000012397766,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13067","display_name":"Geological Modeling and Analysis","score":0.027699999511241913,"subfield":{"id":"https://openalex.org/subfields/1906","display_name":"Geochemistry and Petrology"},"field":{"id":"https://openalex.org/fields/19","display_name":"Earth and Planetary Sciences"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14311","display_name":"Electrical and Electromagnetic Research","score":0.015699999406933784,"subfield":{"id":"https://openalex.org/subfields/3107","display_name":"Atomic and Molecular Physics, and Optics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.7692999839782715},{"id":"https://openalex.org/keywords/prefix","display_name":"Prefix","score":0.5436999797821045},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.4618000090122223},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.41819998621940613},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.41290000081062317},{"id":"https://openalex.org/keywords/cache-algorithms","display_name":"Cache algorithms","score":0.40049999952316284},{"id":"https://openalex.org/keywords/cpu-cache","display_name":"CPU cache","score":0.39660000801086426}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8802000284194946},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.7692999839782715},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6868000030517578},{"id":"https://openalex.org/C141603448","wikidata":"https://www.wikidata.org/wiki/Q134830","display_name":"Prefix","level":2,"score":0.5436999797821045},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4618000090122223},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.41819998621940613},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.41290000081062317},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.40049999952316284},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.39660000801086426},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.3785000145435333},{"id":"https://openalex.org/C190290938","wikidata":"https://www.wikidata.org/wiki/Q387015","display_name":"Trie","level":3,"score":0.3537999987602234},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.31360000371932983},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.30469998717308044},{"id":"https://openalex.org/C136085584","wikidata":"https://www.wikidata.org/wiki/Q910289","display_name":"Overlay","level":2,"score":0.288100004196167},{"id":"https://openalex.org/C141917322","wikidata":"https://www.wikidata.org/wiki/Q1025017","display_name":"Cache coherence","level":5,"score":0.27869999408721924},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.2750000059604645},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.2736000120639801},{"id":"https://openalex.org/C167713795","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"Smart Cache","level":5,"score":0.26409998536109924},{"id":"https://openalex.org/C201148951","wikidata":"https://www.wikidata.org/wiki/Q5015976","display_name":"Cache coloring","level":4,"score":0.25679999589920044},{"id":"https://openalex.org/C51185590","wikidata":"https://www.wikidata.org/wiki/Q1017228","display_name":"Bus sniffing","level":5,"score":0.2531000077724457},{"id":"https://openalex.org/C78766204","wikidata":"https://www.wikidata.org/wiki/Q555032","display_name":"Multi-core processor","level":2,"score":0.25209999084472656}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3749168","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3749168","pdf_url":null,"source":{"id":"https://openalex.org/S4387289859","display_name":"Proceedings of the ACM on Management of Data","issn_l":"2836-6573","issn":["2836-6573"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Management of Data","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6585914614","display_name":null,"funder_award_id":"NO.62325205, 62072230","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":10,"referenced_works":["https://openalex.org/W1967373117","https://openalex.org/W2003549201","https://openalex.org/W3046467528","https://openalex.org/W3112116031","https://openalex.org/W4317898419","https://openalex.org/W4387321091","https://openalex.org/W4393147129","https://openalex.org/W4398150831","https://openalex.org/W4402670433","https://openalex.org/W4414371540"],"related_works":[],"abstract_inverted_index":{"Prompt":[0],"engineering":[1],"techniques":[2],"are":[3],"widely":[4],"used":[5],"to":[6,39,64,162,187,207,249,263],"enhance":[7,233],"the":[8,17,48,56,76,134,208,214,237],"generation":[9],"quality":[10],"of":[11,51,78,136],"large":[12,49],"language":[13],"models":[14],"(LLMs).":[15],"However,":[16],"long":[18],"prompts":[19,30],"significantly":[20,224],"increase":[21],"inference":[22,26,118,230,242],"latency":[23,243,265],"and":[24,55,90,132,167,200,232,244,267],"reduce":[25,168,229],"throughput.":[27,234],"Since":[28],"many":[29],"share":[31],"common":[32],"prefixes,":[33],"prefix":[34,42,52,67,114,137,180,255],"sharing":[35,115,256],"has":[36],"been":[37],"proposed":[38],"reuse":[40],"shared":[41],"KV":[43,53,68,107,144,150,169,182,219],"caches":[44,54,69,151,158,183],"during":[45],"inference.":[46],"Nevertheless,":[47],"number":[50],"limited":[57],"GPU":[58,71,153,164,188,203],"memory":[59,80,161,165,186,204],"capacity":[60,166],"make":[61],"it":[62,260],"impractical":[63],"store":[65],"all":[66],"in":[70,116,159,236],"memory.":[72,189],"This":[73,190],"limitation":[74],"necessitates":[75],"use":[77],"external":[79],"storage":[81],"strategies,":[82],"which":[83,129,147,176],"often":[84],"suffer":[85],"from":[86,152,184],"high":[87],"I/O":[88,199,215],"overhead":[89,216],"frequent":[91],"cache":[92,108,170,220,226],"misses":[93],"with":[94,193,218,252,254],"traditional":[95],"approaches.":[96],"To":[97],"address":[98],"these":[99],"challenges,":[100],"this":[101],"paper":[102],"proposes":[103],"HotPrefix,":[104],"a":[105],"hotness-aware":[106],"scheduling":[109],"framework":[110],"designed":[111],"for":[112,198],"efficient":[113,195],"LLM":[117],"systems.":[119],"HotPrefix":[120,240],"introduces":[121],"three":[122],"core":[123],"innovations:":[124],"(1)":[125],"Dynamic":[126],"Hotness":[127,174],"Tracking,":[128],"dynamically":[130],"monitors":[131],"updates":[133],"hotness":[135],"tree":[138,181],"nodes":[139],"over":[140],"time;":[141],"(2)":[142],"Selective":[143],"Cache":[145],"Admission,":[146],"evaluates":[148],"evicted":[149],"memory,":[154],"retaining":[155],"only":[156],"high-hotness":[157,179],"CPU":[160,185],"expand":[163],"transfer":[171],"overhead;":[172],"(3)":[173],"Promotion,":[175],"periodically":[177],"promotes":[178],"is":[191,205],"combined":[192],"an":[194],"pipeline":[196],"strategy":[197],"computation,":[201],"ensuring":[202],"allocated":[206],"most":[209],"critical":[210],"prefixes":[211],"while":[212],"masking":[213],"associated":[217],"transmission.":[221],"These":[222],"mechanisms":[223],"improve":[225],"hit":[227],"rates,":[228],"latency,":[231],"Implemented":[235],"SGLang":[238],"framework,":[239],"reduces":[241],"increases":[245],"throughput":[246,268],"by":[247],"up":[248,262],"2.25\u00d7":[250],"compared":[251],"vLLM":[253],"enabled.":[257],"Against":[258],"SGLang,":[259],"achieves":[261],"2\u00d7":[264],"reduction":[266],"increase.":[269]},"counts_by_year":[{"year":2026,"cited_by_count":4}],"updated_date":"2026-06-26T08:34:08.712188","created_date":"2025-10-10T00:00:00"}