{"id":"https://openalex.org/W4412887830","doi":"https://doi.org/10.18653/v1/2025.findings-acl.997","title":"KVPR: Efficient LLM Inference with I/O-Aware KV Cache Partial Recomputation","display_name":"KVPR: Efficient LLM Inference with I/O-Aware KV Cache Partial Recomputation","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412887830","doi":"https://doi.org/10.18653/v1/2025.findings-acl.997"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.997","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.997","pdf_url":"https://aclanthology.org/2025.findings-acl.997.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.997.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052218561","display_name":"Chaoyi Jiang","orcid":null},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]},{"id":"https://openalex.org/I2800817003","display_name":"California Southern University","ror":"https://ror.org/058zz0t50","country_code":"US","type":"education","lineage":["https://openalex.org/I2800817003"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chaoyi Jiang","raw_affiliation_strings":["University of Southern California"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Southern California","institution_ids":["https://openalex.org/I2800817003","https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042870819","display_name":"Lei Gao","orcid":"https://orcid.org/0000-0003-3916-6641"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]},{"id":"https://openalex.org/I2800817003","display_name":"California Southern University","ror":"https://ror.org/058zz0t50","country_code":"US","type":"education","lineage":["https://openalex.org/I2800817003"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lei Gao","raw_affiliation_strings":["University of Southern California"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Southern California","institution_ids":["https://openalex.org/I2800817003","https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104435472","display_name":"Hossein Entezari Zarch","orcid":null},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]},{"id":"https://openalex.org/I2800817003","display_name":"California Southern University","ror":"https://ror.org/058zz0t50","country_code":"US","type":"education","lineage":["https://openalex.org/I2800817003"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hossein Entezari Zarch","raw_affiliation_strings":["University of Southern California"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Southern California","institution_ids":["https://openalex.org/I2800817003","https://openalex.org/I1174212"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5018033573","display_name":"Murali Annavaram","orcid":"https://orcid.org/0000-0002-4633-6867"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Murali Annavaram","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"19474","last_page":"19488"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.9934999942779541,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10901","display_name":"Advanced Data Compression Techniques","score":0.9871000051498413,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7884581685066223},{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.718416154384613},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.6040244102478027},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5973573923110962},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.40731775760650635},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.18646934628486633}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7884581685066223},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.718416154384613},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6040244102478027},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5973573923110962},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.40731775760650635},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.18646934628486633}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.997","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.997","pdf_url":"https://aclanthology.org/2025.findings-acl.997.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.997","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.997","pdf_url":"https://aclanthology.org/2025.findings-acl.997.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412887830.pdf","grobid_xml":"https://content.openalex.org/works/W4412887830.grobid-xml"},"referenced_works_count":1,"referenced_works":["https://openalex.org/W4281758439"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Inference":[0],"for":[1,30,36],"Large":[2],"Language":[3],"Models":[4],"(LLMs)":[5],"is":[6,18,51,177,200,260],"computationally":[7],"demanding.To":[8],"reduce":[9],"the":[10,27,33,37,44,66,69,73,77,118,121,126,143,154,159,163,166,170,174,181,221,235],"cost":[11],"of":[12,46,72,120,150,173,223],"auto-regressive":[13],"decoding,":[14],"Key-Value":[15],"(KV)":[16],"cache":[17,39,55,123,161,176,189],"used":[19],"to":[20,52,56,68,83,191,219,232,245,256],"store":[21],"intermediate":[22],"activations,":[23,151],"which":[24,59,153],"significantly":[25],"lowers":[26],"computational":[28],"overhead":[29],"token":[31],"generation.However,":[32],"memory":[34,62],"required":[35],"KV":[38,54,122,160,168,175,188],"grows":[40,124],"rapidly,":[41],"often":[42],"exceeding":[43],"capacity":[45],"GPU":[47,61,89,127,155,164,185,194],"memory.A":[48],"cost-effective":[49],"alternative":[50],"offload":[53],"CPU":[57,78,109,144],"memory,":[58],"alleviates":[60],"pressure,":[63],"but":[64,98],"shifts":[65],"bottleneck":[67],"limited":[70],"bandwidth":[71],"PCIe":[74,112],"connection":[75],"between":[76],"and":[79,106,196,212,225,228,249],"GPU.Existing":[80],"methods":[81],"attempt":[82],"address":[84],"these":[85],"issues":[86],"by":[87,102,203],"overlapping":[88,111],"computation":[90,224],"with":[91,187],"I/O":[92],"or":[93],"employing":[94],"CPU-GPU":[95],"heterogeneous":[96],"execution,":[97],"they":[99],"are":[100],"hindered":[101],"excessive":[103],"data":[104],"movement":[105],"dependence":[107],"on":[108],"capabilities.Fully":[110],"communication":[113,226],"latency":[114,248],"gets":[115],"challenging":[116],"as":[117],"size":[119],"and/or":[125],"compute":[128],"capabilities":[129],"increase.In":[130],"this":[131],"paper,":[132],"we":[133],"introduce":[134],"KVPR,":[135],"an":[136],"efficient":[137],"I/O-aware":[138],"LLM":[139],"inference":[140,198],"method":[141],"where":[142],"first":[145],"transfers":[146],"a":[147,205,216,229],"partial":[148,167],"set":[149],"from":[152,180],"can":[156],"start":[157],"recomputing":[158],"values.While":[162],"recomputes":[165],"cache,":[169],"remaining":[171],"portion":[172],"transferred":[178],"concurrently":[179],"CPU.This":[182],"approach":[183],"overlaps":[184],"recomputation":[186],"transfer":[190],"minimize":[192],"idle":[193],"time":[195],"maximize":[197],"performance.KVPR":[199],"fully":[201],"automated":[202],"integrating":[204],"profiler":[206],"module":[207,218,231],"that":[208,241],"utilizes":[209],"input":[210],"characteristics":[211],"system":[213],"hardware":[214],"information,":[215],"scheduler":[217],"optimize":[220],"distribution":[222],"workloads,":[227],"runtime":[230],"efficiently":[233],"execute":[234],"derived":[236],"execution":[237],"plan.Experimental":[238],"results":[239],"show":[240],"KVPR":[242],"achieves":[243],"up":[244],"35.8%":[246],"lower":[247],"46.2%":[250],"higher":[251],"throughput":[252],"during":[253],"decoding":[254],"compared":[255],"stateof-the-art":[257],"approaches.The":[258],"code":[259],"available":[261],"at":[262],"https://github.com/chaoyij/KVPR.":[263]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-17T09:13:05.818461","created_date":"2025-10-10T00:00:00"}