{"id":"https://openalex.org/W7124152561","doi":"https://doi.org/10.1109/icpads67057.2025.11323033","title":"Dispenser: Hierarchical KV Cache Management for Efficient LLM Generative Inference","display_name":"Dispenser: Hierarchical KV Cache Management for Efficient LLM Generative Inference","publication_year":2025,"publication_date":"2025-12-14","ids":{"openalex":"https://openalex.org/W7124152561","doi":"https://doi.org/10.1109/icpads67057.2025.11323033"},"language":null,"primary_location":{"id":"doi:10.1109/icpads67057.2025.11323033","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icpads67057.2025.11323033","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 31th International Conference on Parallel and Distributed Systems (ICPADS)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5123011141","display_name":"Beiquan Cao","orcid":null},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Beiquan Cao","raw_affiliation_strings":["School of EECS, Peking University,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of EECS, Peking University,Beijing,China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088635219","display_name":"Kaigui Bian","orcid":null},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kaigui Bian","raw_affiliation_strings":["School of Computer Science, Peking University,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science, Peking University,Beijing,China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123046976","display_name":"Guojie Luo","orcid":null},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guojie Luo","raw_affiliation_strings":["School of Computer Science, Peking University,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science, Peking University,Beijing,China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5123034593","display_name":"Joongheon Kim","orcid":null},"institutions":[{"id":"https://openalex.org/I197347611","display_name":"Korea University","ror":"https://ror.org/047dqcg40","country_code":"KR","type":"education","lineage":["https://openalex.org/I197347611"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Joongheon Kim","raw_affiliation_strings":["School of Electrical Engineering, Korea University,Seoul,Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Electrical Engineering, Korea University,Seoul,Republic of Korea","institution_ids":["https://openalex.org/I197347611"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.164900004863739,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.164900004863739,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.13590000569820404,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.09860000014305115,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.7275999784469604},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6481999754905701},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.5509999990463257},{"id":"https://openalex.org/keywords/cache-algorithms","display_name":"Cache algorithms","score":0.5508999824523926},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.3885999917984009},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.3856000006198883},{"id":"https://openalex.org/keywords/cache-pollution","display_name":"Cache pollution","score":0.35499998927116394},{"id":"https://openalex.org/keywords/context-switch","display_name":"Context switch","score":0.3547999858856201}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.864300012588501},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.7275999784469604},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6481999754905701},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.5509999990463257},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.5508999824523926},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.45590001344680786},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.3885999917984009},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.3856000006198883},{"id":"https://openalex.org/C113166858","wikidata":"https://www.wikidata.org/wiki/Q5015981","display_name":"Cache pollution","level":5,"score":0.35499998927116394},{"id":"https://openalex.org/C53833338","wikidata":"https://www.wikidata.org/wiki/Q1061424","display_name":"Context switch","level":2,"score":0.3547999858856201},{"id":"https://openalex.org/C42812","wikidata":"https://www.wikidata.org/wiki/Q1082910","display_name":"Partition (number theory)","level":2,"score":0.3513000011444092},{"id":"https://openalex.org/C59687516","wikidata":"https://www.wikidata.org/wiki/Q5015938","display_name":"Cache-oblivious algorithm","level":5,"score":0.33970001339912415},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.33719998598098755},{"id":"https://openalex.org/C201148951","wikidata":"https://www.wikidata.org/wiki/Q5015976","display_name":"Cache coloring","level":4,"score":0.3319000005722046},{"id":"https://openalex.org/C2778869765","wikidata":"https://www.wikidata.org/wiki/Q6028363","display_name":"Inefficiency","level":2,"score":0.32179999351501465},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.3190999925136566},{"id":"https://openalex.org/C25536678","wikidata":"https://www.wikidata.org/wiki/Q5015977","display_name":"Cache invalidation","level":5,"score":0.29660001397132874},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2955000102519989},{"id":"https://openalex.org/C124304363","wikidata":"https://www.wikidata.org/wiki/Q673661","display_name":"Abstraction","level":2,"score":0.2734000086784363},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.2703000009059906},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.26339998841285706},{"id":"https://openalex.org/C2781357197","wikidata":"https://www.wikidata.org/wiki/Q5757597","display_name":"High memory","level":2,"score":0.2581999897956848}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icpads67057.2025.11323033","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icpads67057.2025.11323033","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 31th International Conference on Parallel and Distributed Systems (ICPADS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G661206769","display_name":null,"funder_award_id":"U24A20235,62032003","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W2070246124","https://openalex.org/W2165533158","https://openalex.org/W2889787757","https://openalex.org/W2963204221","https://openalex.org/W2963339397","https://openalex.org/W2963963993","https://openalex.org/W2989743967","https://openalex.org/W3115947671","https://openalex.org/W3161820423","https://openalex.org/W3170490008","https://openalex.org/W3171639395","https://openalex.org/W3190126809","https://openalex.org/W4310494058","https://openalex.org/W4387321091","https://openalex.org/W4388979610","https://openalex.org/W4402671766","https://openalex.org/W4402671799","https://openalex.org/W4411403450","https://openalex.org/W4412889578"],"related_works":[],"abstract_inverted_index":{"As":[0],"context":[1],"windows":[2],"expands,":[3],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"are":[8],"being":[9],"more":[10],"capable":[11],"of":[12,43,56],"handling":[13],"long-context":[14,17],"tasks.":[15],"However,":[16,77],"inference":[18,186],"is":[19,84],"hindered":[20],"by":[21,87,119,188],"high":[22],"latency":[23,187],"and":[24,50,69,71,93,104,133,138,159],"memory":[25,102],"usage":[26],"due":[27],"to":[28,157,176],"the":[29,44,57,63,125,161,166],"linearly":[30],"growing":[31],"Key-Value":[32],"(KV)":[33],"cache.":[34,59],"Prior":[35],"work":[36],"has":[37],"shown":[38],"that":[39,172],"a":[40,112,153,177],"dynamic":[41],"subset":[42],"KV":[45,64,114,126],"cache":[46,65,115,127],"dominates":[47],"attention":[48],"outcomes":[49],"selectively":[51],"loads":[52],"these":[53,107],"subsets":[54],"instead":[55],"full":[58],"These":[60],"methods":[61],"partition":[62],"into":[66,136,144],"segments":[67],"(block-wise":[68],"token-wise)":[70],"identify":[72,158],"critical":[73],"ones":[74],"for":[75,78,94,165],"recall.":[76],"block-wise":[79],"ones,":[80,96],"their":[81],"recall":[82],"accuracy":[83,200],"often":[85],"compromised":[86],"semantic":[88,131],"inconsistencies":[89],"among":[90],"consecutive":[91],"tokens,":[92],"token-wise":[95],"they":[97],"suffer":[98],"from":[99],"inefficiency":[100],"in":[101,183],"management":[103,116],"offloading.":[105],"To":[106],"ends,":[108],"we":[109],"propose":[110],"Dispenser,":[111],"hierarchical":[113],"algorithm":[117],"inspired":[118],"multi-level":[120],"page":[121],"tables.":[122],"Dispenser":[123,173],"organizes":[124],"based":[128],"on":[129,193],"token":[130],"similarity":[132],"partitions":[134],"it":[135],"pages":[137],"blocks,":[139],"which":[140],"enables":[141],"straightforward":[142],"integration":[143],"modern":[145],"serving":[146],"engines":[147],"such":[148],"as":[149],"vLLM.":[150],"It":[151],"employs":[152],"two-level":[154],"lookup":[155],"mechanism":[156],"load":[160],"most":[162],"relevant":[163],"blocks":[164],"current":[167],"query.":[168],"Experimental":[169],"results":[170],"demonstrate":[171],"achieves":[174],"up":[175],"<tex":[178,189],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[179,190],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$5.75":[180],"\\times$</tex>":[181,192],"speedup":[182],"self-attention,":[184],"reducing":[185],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$2.1":[191],"tasks":[194],"with":[195,198],"long":[196],"dependencies":[197],"negligible":[199],"loss.":[201]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-01-15T00:00:00"}