{"id":"https://openalex.org/W7168020287","doi":"https://doi.org/10.1145/3806645.3807819","title":"PKAS: Predictive KVCache-Aware Scheduling for Faster LLM and Transformer Inferences","display_name":"PKAS: Predictive KVCache-Aware Scheduling for Faster LLM and Transformer Inferences","publication_year":2026,"publication_date":"2026-07-11","ids":{"openalex":"https://openalex.org/W7168020287","doi":"https://doi.org/10.1145/3806645.3807819"},"language":null,"primary_location":{"id":"doi:10.1145/3806645.3807819","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3806645.3807819","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 35th International Symposium on High-Performance Parallel and Distributed Computing","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3806645.3807819","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140468215","display_name":"Jie Ye","orcid":"https://orcid.org/0009-0003-3985-7896"},"institutions":[{"id":"https://openalex.org/I180949307","display_name":"Illinois Institute of Technology","ror":"https://ror.org/037t3ry66","country_code":"US","type":"education","lineage":["https://openalex.org/I180949307"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jie Ye","raw_affiliation_strings":["Illinois Institute of Technology, Chicago, Illinois, USA"],"raw_orcid":"https://orcid.org/0009-0003-3985-7896","affiliations":[{"raw_affiliation_string":"Illinois Institute of Technology, Chicago, Illinois, USA","institution_ids":["https://openalex.org/I180949307"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049218913","display_name":"Avinash Maurya","orcid":"https://orcid.org/0000-0002-8200-0148"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Avinash Maurya","raw_affiliation_strings":["Argonne National Laboratory, Lemont, USA"],"raw_orcid":"https://orcid.org/0000-0002-8200-0148","affiliations":[{"raw_affiliation_string":"Argonne National Laboratory, Lemont, USA","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015607198","display_name":"Krishna Teja Chitty-Venkata","orcid":"https://orcid.org/0000-0002-3027-1915"},"institutions":[{"id":"https://openalex.org/I1328219102","display_name":"Red Hat (United States)","ror":"https://ror.org/03ncs3316","country_code":"US","type":"company","lineage":["https://openalex.org/I1328219102"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Krishna Teja Chitty-Venkata","raw_affiliation_strings":["Red Hat, Inc., Boston, USA"],"raw_orcid":"https://orcid.org/0000-0002-3027-1915","affiliations":[{"raw_affiliation_string":"Red Hat, Inc., Boston, USA","institution_ids":["https://openalex.org/I1328219102"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085745891","display_name":"Bogdan Nicolae","orcid":"https://orcid.org/0000-0002-0661-7509"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bogdan Nicolae","raw_affiliation_strings":["Argonne National Laboratory, Lemont, USA"],"raw_orcid":"https://orcid.org/0000-0002-0661-7509","affiliations":[{"raw_affiliation_string":"Argonne National Laboratory, Lemont, USA","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076884394","display_name":"Anthony Kougkas","orcid":"https://orcid.org/0000-0003-3943-663X"},"institutions":[{"id":"https://openalex.org/I180949307","display_name":"Illinois Institute of Technology","ror":"https://ror.org/037t3ry66","country_code":"US","type":"education","lineage":["https://openalex.org/I180949307"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anthony Kougkas","raw_affiliation_strings":["Illinois Institute of Technology, Chicago, Illinois, USA"],"raw_orcid":"https://orcid.org/0000-0003-3943-663X","affiliations":[{"raw_affiliation_string":"Illinois Institute of Technology, Chicago, Illinois, USA","institution_ids":["https://openalex.org/I180949307"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057665558","display_name":"Xian\u2010He Sun","orcid":"https://orcid.org/0000-0002-1093-0792"},"institutions":[{"id":"https://openalex.org/I180949307","display_name":"Illinois Institute of Technology","ror":"https://ror.org/037t3ry66","country_code":"US","type":"education","lineage":["https://openalex.org/I180949307"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xian-He Sun","raw_affiliation_strings":["Illinois Institute of Technology, Chicago, Illinois, USA"],"raw_orcid":"https://orcid.org/0000-0002-1093-0792","affiliations":[{"raw_affiliation_string":"Illinois Institute of Technology, Chicago, Illinois, USA","institution_ids":["https://openalex.org/I180949307"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.93137698,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"402","last_page":"414"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.42969998717308044,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.42969998717308044,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.2363000065088272,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.05460000038146973,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.7925999760627747},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.5327000021934509},{"id":"https://openalex.org/keywords/cache-coloring","display_name":"Cache coloring","score":0.4733999967575073},{"id":"https://openalex.org/keywords/interleaving","display_name":"Interleaving","score":0.4472000002861023},{"id":"https://openalex.org/keywords/cache-algorithms","display_name":"Cache algorithms","score":0.42309999465942383},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.4187000095844269},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.41769999265670776},{"id":"https://openalex.org/keywords/offset","display_name":"Offset (computer science)","score":0.36640000343322754}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.86080002784729},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.7925999760627747},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6015999913215637},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.5327000021934509},{"id":"https://openalex.org/C201148951","wikidata":"https://www.wikidata.org/wiki/Q5015976","display_name":"Cache coloring","level":4,"score":0.4733999967575073},{"id":"https://openalex.org/C28034677","wikidata":"https://www.wikidata.org/wiki/Q17092530","display_name":"Interleaving","level":2,"score":0.4472000002861023},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.42309999465942383},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4187000095844269},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.41769999265670776},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.36640000343322754},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.3467000126838684},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.3434000015258789},{"id":"https://openalex.org/C113166858","wikidata":"https://www.wikidata.org/wiki/Q5015981","display_name":"Cache pollution","level":5,"score":0.34130001068115234},{"id":"https://openalex.org/C167713795","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"Smart Cache","level":5,"score":0.31200000643730164},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.30820000171661377},{"id":"https://openalex.org/C177950962","wikidata":"https://www.wikidata.org/wiki/Q10997658","display_name":"Non-volatile memory","level":2,"score":0.2953999936580658},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2881999909877777},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.2874999940395355},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.2863999903202057},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.2849999964237213},{"id":"https://openalex.org/C2989134064","wikidata":"https://www.wikidata.org/wiki/Q288510","display_name":"Execution time","level":2,"score":0.28450000286102295},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.26919999718666077},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.263700008392334},{"id":"https://openalex.org/C2778915421","wikidata":"https://www.wikidata.org/wiki/Q3643177","display_name":"Performance improvement","level":2,"score":0.25679999589920044}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3806645.3807819","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3806645.3807819","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 35th International Symposium on High-Performance Parallel and Distributed Computing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3806645.3807819","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3806645.3807819","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 35th International Symposium on High-Performance Parallel and Distributed Computing","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.5305851697921753,"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W1987034518","https://openalex.org/W3034368386","https://openalex.org/W4387321091","https://openalex.org/W4390263770","https://openalex.org/W4396534784","https://openalex.org/W4402670328","https://openalex.org/W4402670999","https://openalex.org/W4402671766","https://openalex.org/W4404401017","https://openalex.org/W4412610757","https://openalex.org/W4414371540","https://openalex.org/W4415797367","https://openalex.org/W4415797413","https://openalex.org/W7133235396","https://openalex.org/W7133238857"],"related_works":[],"abstract_inverted_index":{"With":[0],"rising":[1],"popularity":[2],"of":[3,10,19,80,129],"LLMs,":[4],"the":[5,20,24,34,61,78,102,111,166,215],"performance,":[6],"scalability,":[7],"and":[8,33,56,127,135,164,187,195,206],"resource-efficiency":[9],"inferences":[11],"become":[12],"a":[13,140,155],"crucial":[14],"challenge.":[15],"The":[16],"core":[17],"part":[18],"inference":[21,67],"process":[22],"is":[23,225],"KV":[25,48,104,119,142,161,184,222],"cache,":[26],"which":[27,122],"avoids":[28],"recomputing":[29],"intermediate":[30],"attention":[31],"states,":[32],"batching":[35,71,181],"strategy":[36],"that":[37,198],"batches":[38],"multiple":[39],"requests":[40,82,89,99],"per":[41],"forward":[42],"pass":[43],"to":[44,72,108,146,158,202,211],"leverage":[45],"GPU":[46,63,74],"parallelism.":[47],"cache":[49,105,120,162,185,223],"memory":[50,64,106],"grows":[51],"linearly":[52],"with":[53,86,173,214],"sequence":[54],"length":[55],"batch":[57],"sizes,":[58],"easily":[59],"exceeding":[60],"limited":[62],"capacity.":[65],"State-of-the-art":[66],"runtimes":[68],"use":[69],"continuous":[70],"maximize":[73],"utilization":[75,163],"by":[76,150],"interleaving":[77],"processing":[79],"new":[81,169],"(i.e.,":[83,90],"prefill":[84,98],"requests)":[85],"ongoing":[87],"generation":[88],"decode":[91,112],"requests).":[92],"However,":[93],"existing":[94],"schedulers":[95],"greedily":[96],"admit":[97],"without":[100],"considering":[101],"future":[103,160],"required":[107],"successfully":[109],"run":[110],"phases.":[113],"This":[114],"shortsighted":[115],"approach":[116],"causes":[117],"frequent":[118],"overflows,":[121],"in":[123],"turn":[124],"trigger":[125],"preemption":[126],"recomputation":[128],"requests,":[130],"severely":[131],"degrading":[132],"both":[133],"throughput":[134,205],"latency.":[136],"We":[137],"propose":[138],"PKAS,":[139],"Predictive":[141],"Cache-Aware":[143],"Scheduling":[144],"algorithm":[145],"mitigate":[147],"this":[148],"inefficiency":[149],"reducing":[151,189],"preemptions.":[152,190],"PKAS":[153,177,199],"uses":[154],"low-overhead":[156],"technique":[157],"simulate":[159],"guide":[165],"admissibility":[167],"for":[168],"request":[170],"candidates.":[171],"Combined":[172],"lightweight":[174],"output-length":[175],"predictions,":[176],"can":[178],"make":[179],"better":[180],"decisions,":[182],"preventing":[183],"overflows":[186],"drastically":[188],"Evaluations":[191],"on":[192,218],"diverse":[193],"models":[194],"workloads":[196,220],"show":[197],"achieves":[200],"up":[201],"7.34x":[203],"higher":[204],"8x":[207],"lower":[208],"latency":[209],"compared":[210],"state-of-the-art":[212],"scheduling,":[213],"largest":[216],"gains":[217],"long-context":[219],"where":[221],"pressure":[224],"high.":[226]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-07-12T00:00:00"}