{"id":"https://openalex.org/W4414735889","doi":"https://doi.org/10.1145/3731569.3764810","title":"DiffKV: Differentiated Memory Management for Large Language Models with Parallel KV Compaction","display_name":"DiffKV: Differentiated Memory Management for Large Language Models with Parallel KV Compaction","publication_year":2025,"publication_date":"2025-10-01","ids":{"openalex":"https://openalex.org/W4414735889","doi":"https://doi.org/10.1145/3731569.3764810"},"language":"en","primary_location":{"id":"doi:10.1145/3731569.3764810","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3731569.3764810","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM SIGOPS 31st Symposium on Operating Systems Principles","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3731569.3764810","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5087483400","display_name":"Yanqi Zhang","orcid":"https://orcid.org/0000-0003-0130-1502"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanqi Zhang","raw_affiliation_strings":["Huawei, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-0130-1502","affiliations":[{"raw_affiliation_string":"Huawei, Beijing, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053029277","display_name":"Yuwei Hu","orcid":null},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuwei Hu","raw_affiliation_strings":["Huawei, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0008-6246-4253","affiliations":[{"raw_affiliation_string":"Huawei, Beijing, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Runyuan Zhao","orcid":"https://orcid.org/0009-0003-1305-901X"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Runyuan Zhao","raw_affiliation_strings":["Huawei, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0003-1305-901X","affiliations":[{"raw_affiliation_string":"Huawei, Beijing, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068489266","display_name":"John C. S. Lui","orcid":"https://orcid.org/0000-0001-7466-0384"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"John C. S. Lui","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0001-7466-0384","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100406215","display_name":"Haibo Chen","orcid":"https://orcid.org/0000-0002-9720-0361"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haibo Chen","raw_affiliation_strings":["Shanghai JiaoTong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-9720-0361","affiliations":[{"raw_affiliation_string":"Shanghai JiaoTong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"431","last_page":"445"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9729999899864197,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9078999757766724,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.7253999710083008},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.5006999969482422},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.47620001435279846},{"id":"https://openalex.org/keywords/cache-only-memory-architecture","display_name":"Cache-only memory architecture","score":0.46059998869895935},{"id":"https://openalex.org/keywords/cache-pollution","display_name":"Cache pollution","score":0.4447000026702881},{"id":"https://openalex.org/keywords/cache-algorithms","display_name":"Cache algorithms","score":0.4250999987125397},{"id":"https://openalex.org/keywords/cache-coloring","display_name":"Cache coloring","score":0.421099990606308},{"id":"https://openalex.org/keywords/memory-management","display_name":"Memory management","score":0.4117000102996826},{"id":"https://openalex.org/keywords/cpu-cache","display_name":"CPU cache","score":0.3668000102043152}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8579000234603882},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.7253999710083008},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5277000069618225},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.5006999969482422},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.47620001435279846},{"id":"https://openalex.org/C3720319","wikidata":"https://www.wikidata.org/wiki/Q5015937","display_name":"Cache-only memory architecture","level":5,"score":0.46059998869895935},{"id":"https://openalex.org/C113166858","wikidata":"https://www.wikidata.org/wiki/Q5015981","display_name":"Cache pollution","level":5,"score":0.4447000026702881},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.4250999987125397},{"id":"https://openalex.org/C201148951","wikidata":"https://www.wikidata.org/wiki/Q5015976","display_name":"Cache coloring","level":4,"score":0.421099990606308},{"id":"https://openalex.org/C176649486","wikidata":"https://www.wikidata.org/wiki/Q2308807","display_name":"Memory management","level":3,"score":0.4117000102996826},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.3668000102043152},{"id":"https://openalex.org/C25536678","wikidata":"https://www.wikidata.org/wiki/Q5015977","display_name":"Cache invalidation","level":5,"score":0.34689998626708984},{"id":"https://openalex.org/C76399640","wikidata":"https://www.wikidata.org/wiki/Q189401","display_name":"Virtual memory","level":4,"score":0.32429999113082886},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.322299987077713},{"id":"https://openalex.org/C41036726","wikidata":"https://www.wikidata.org/wiki/Q844824","display_name":"Physical address","level":3,"score":0.3010999858379364},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.2928999960422516},{"id":"https://openalex.org/C133371097","wikidata":"https://www.wikidata.org/wiki/Q868014","display_name":"Non-uniform memory access","level":5,"score":0.2892000079154968},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.2865999937057495},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.28360000252723694},{"id":"https://openalex.org/C2781357197","wikidata":"https://www.wikidata.org/wiki/Q5757597","display_name":"High memory","level":2,"score":0.28200000524520874},{"id":"https://openalex.org/C12186640","wikidata":"https://www.wikidata.org/wiki/Q6815743","display_name":"Memory model","level":3,"score":0.2732999920845032},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.266400009393692},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.26179999113082886},{"id":"https://openalex.org/C138101251","wikidata":"https://www.wikidata.org/wiki/Q213092","display_name":"Thread (computing)","level":2,"score":0.2551000118255615}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3731569.3764810","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3731569.3764810","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM SIGOPS 31st Symposium on Operating Systems Principles","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3731569.3764810","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3731569.3764810","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM SIGOPS 31st Symposium on Operating Systems Principles","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W95608104","https://openalex.org/W1482147200","https://openalex.org/W2912616132","https://openalex.org/W2915041608","https://openalex.org/W4205983429","https://openalex.org/W4213153339","https://openalex.org/W4226064176","https://openalex.org/W4245299501","https://openalex.org/W4254240460","https://openalex.org/W4254620533","https://openalex.org/W4281758439","https://openalex.org/W4283768109","https://openalex.org/W4308760184","https://openalex.org/W4321637333","https://openalex.org/W4378718572","https://openalex.org/W4382319938","https://openalex.org/W4387302738","https://openalex.org/W4388093177","https://openalex.org/W4392867129","https://openalex.org/W4398796846"],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,178],"(LLMs)":[3],"demonstrate":[4],"remarkable":[5],"capabilities":[6],"but":[7],"face":[8],"substantial":[9],"serving":[10],"costs":[11],"due":[12],"to":[13,40,188,195,214],"their":[14],"high":[15],"memory":[16,120,134,144,150],"demands,":[17],"with":[18,197],"the":[19,51,55,84,88,99,106,161,175,190],"key-value":[20],"(KV)":[21],"cache":[22,29,60,75,163,192],"being":[23],"a":[24,69],"primary":[25],"bottleneck.":[26],"State-of-the-art":[27],"KV":[28,59,74,85,162,191],"compression":[30,76],"techniques,":[31],"such":[32,64],"as":[33],"quantization":[34],"and":[35,43,45,93,104,126,206,209],"pruning,":[36],"apply":[37],"uniform":[38],"treatment":[39],"both":[41],"keys":[42,92],"values,":[44],"discard":[46],"unimportant":[47],"tokens":[48],"entirely,":[49],"overlooking":[50],"fine-grained":[52],"distinctions":[53],"in":[54,83,155,160],"significance":[56],"of":[57,81,91,102,116,183],"individual":[58],"components.":[61],"To":[62,136],"address":[63,137],"limitations,":[65],"we":[66],"introduce":[67,118],"DiffKV,":[68],"novel":[70],"framework":[71],"for":[72,133],"efficient":[73],"that":[77,146,179],"exploits":[78],"three":[79],"levels":[80,115],"differentiation":[82,117],"cache:":[86],"(1)":[87],"differing":[89],"impact":[90],"values":[94],"on":[95,170,200],"attention":[96,112,127],"computation,":[97],"(2)":[98],"varying":[100],"importance":[101],"tokens,":[103],"(3)":[105],"diverse":[107],"dynamic":[108],"sparsity":[109,159],"patterns":[110,122],"across":[111,123],"heads.":[113],"These":[114],"irregular":[119],"usage":[121],"different":[124],"requests":[125],"heads,":[128],"posing":[129],"significant":[130],"scalability":[131],"challenges":[132],"management.":[135],"these":[138],"challenges,":[139],"DiffKV":[140,169,185],"proposes":[141],"an":[142],"on-GPU":[143],"manager":[145],"compacts":[147],"fragmented":[148],"free":[149],"list":[151],"into":[152,164],"contiguous":[153],"regions":[154],"parallel,":[156],"effectively":[157],"translating":[158],"performance":[165],"gains.":[166],"We":[167],"evaluate":[168],"several":[171],"mainstream":[172],"LLMs,":[173],"including":[174],"emerging":[176],"thinking":[177],"generate":[180],"extended":[181],"chains":[182],"thought.":[184],"is":[186],"able":[187],"compress":[189],"by":[193,212],"2.7\u00d7":[194],"5.7\u00d7":[196],"near-lossless":[198],"accuracy":[199],"complex":[201],"workloads":[202],"requiring":[203],"sophisticated":[204],"reasoning":[205],"long-generation":[207],"capabilities,":[208],"enhances":[210],"throughput":[211],"1.9\u00d7":[213],"5.4\u00d7.":[215]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}