{"id":"https://openalex.org/W7124167420","doi":"https://doi.org/10.1109/icpads67057.2025.11323227","title":"Optimizing Distributed LLM Serving through Request Scheduling and Key-Value Cache Sharing","display_name":"Optimizing Distributed LLM Serving through Request Scheduling and Key-Value Cache Sharing","publication_year":2025,"publication_date":"2025-12-14","ids":{"openalex":"https://openalex.org/W7124167420","doi":"https://doi.org/10.1109/icpads67057.2025.11323227"},"language":null,"primary_location":{"id":"doi:10.1109/icpads67057.2025.11323227","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icpads67057.2025.11323227","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 31th International Conference on Parallel and Distributed Systems (ICPADS)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5109629141","display_name":"Hongye Jiang","orcid":null},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]},{"id":"https://openalex.org/I4392021250","display_name":"State Key Laboratory of Networking and Switching Technology","ror":"https://ror.org/00qtv5q45","country_code":null,"type":"facility","lineage":["https://openalex.org/I139759216","https://openalex.org/I4392021250"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongye Jiang","raw_affiliation_strings":["Beijing University of Posts and Telecommunications,State Key Laboratory of Networking and Switching Technology,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing University of Posts and Telecommunications,State Key Laboratory of Networking and Switching Technology,Beijing,China","institution_ids":["https://openalex.org/I139759216","https://openalex.org/I4392021250"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040034745","display_name":"M. Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]},{"id":"https://openalex.org/I4392021250","display_name":"State Key Laboratory of Networking and Switching Technology","ror":"https://ror.org/00qtv5q45","country_code":null,"type":"facility","lineage":["https://openalex.org/I139759216","https://openalex.org/I4392021250"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mu Wang","raw_affiliation_strings":["Beijing University of Posts and Telecommunications,State Key Laboratory of Networking and Switching Technology,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing University of Posts and Telecommunications,State Key Laboratory of Networking and Switching Technology,Beijing,China","institution_ids":["https://openalex.org/I139759216","https://openalex.org/I4392021250"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034995427","display_name":"Su Yao","orcid":"https://orcid.org/0000-0001-5165-2787"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Su Yao","raw_affiliation_strings":["Tsinghua University,Beijing National Research Center for Information Science and Technology,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,Beijing National Research Center for Information Science and Technology,Beijing,China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123036107","display_name":"Cui Ting","orcid":null},"institutions":[{"id":"https://openalex.org/I201459984","display_name":"Guangdong University Of Finances and Economics","ror":"https://ror.org/0459pv085","country_code":"CN","type":"education","lineage":["https://openalex.org/I201459984"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cui Ting","raw_affiliation_strings":["Institute of National Security and Development Studies, Guangdong University of Finance and Economics,Guangdong,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of National Security and Development Studies, Guangdong University of Finance and Economics,Guangdong,China","institution_ids":["https://openalex.org/I201459984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123050010","display_name":"Ziwei Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ziwei Li","raw_affiliation_strings":["Zhongguancun Laboratory,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhongguancun Laboratory,Beijing,China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069646989","display_name":"Changqiao Xu","orcid":"https://orcid.org/0000-0003-1467-1086"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]},{"id":"https://openalex.org/I4392021250","display_name":"State Key Laboratory of Networking and Switching Technology","ror":"https://ror.org/00qtv5q45","country_code":null,"type":"facility","lineage":["https://openalex.org/I139759216","https://openalex.org/I4392021250"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Changqiao Xu","raw_affiliation_strings":["Beijing University of Posts and Telecommunications,State Key Laboratory of Networking and Switching Technology,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing University of Posts and Telecommunications,State Key Laboratory of Networking and Switching Technology,Beijing,China","institution_ids":["https://openalex.org/I139759216","https://openalex.org/I4392021250"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2687999904155731,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2687999904155731,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.22949999570846558,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.10140000283718109,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.6517999768257141},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.6439999938011169},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.6331999897956848},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.5855000019073486},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.508400022983551},{"id":"https://openalex.org/keywords/shared-resource","display_name":"Shared resource","score":0.3560999929904938},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.34779998660087585}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8949000239372253},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.6517999768257141},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.6439999938011169},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.6331999897956848},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.5855000019073486},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5282999873161316},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.508400022983551},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.46790000796318054},{"id":"https://openalex.org/C51332947","wikidata":"https://www.wikidata.org/wiki/Q1172305","display_name":"Shared resource","level":2,"score":0.3560999929904938},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.34779998660087585},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.32589998841285706},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.2948000133037567},{"id":"https://openalex.org/C28034677","wikidata":"https://www.wikidata.org/wiki/Q17092530","display_name":"Interleaving","level":2,"score":0.2935999929904938},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.2912999987602234},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.2888999879360199},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.2782000005245209},{"id":"https://openalex.org/C67953723","wikidata":"https://www.wikidata.org/wiki/Q192525","display_name":"Workstation","level":2,"score":0.27790001034736633},{"id":"https://openalex.org/C2777632111","wikidata":"https://www.wikidata.org/wiki/Q1937518","display_name":"Reservation","level":2,"score":0.27489998936653137},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.26429998874664307}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icpads67057.2025.11323227","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icpads67057.2025.11323227","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 31th International Conference on Parallel and Distributed Systems (ICPADS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.4931802749633789}],"awards":[{"id":"https://openalex.org/G645602424","display_name":null,"funder_award_id":"2024YFC3015300","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G7399245659","display_name":null,"funder_award_id":"62471066","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":3,"referenced_works":["https://openalex.org/W2996600026","https://openalex.org/W3006119691","https://openalex.org/W4387321091"],"related_works":[],"abstract_inverted_index":{"The":[0,149],"widespread":[1],"deployment":[2],"of":[3,18,34,155,191,219],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"is":[8],"often":[9],"constrained":[10],"by":[11,89,165,174],"the":[12,19,31,97,103,129,152,161,171,198,215],"significant":[13],"computational":[14],"and":[15,61,93,121,134,169,210,217],"memory":[16],"demands":[17],"inference":[20],"process.":[21],"A":[22],"critical":[23],"bottleneck":[24],"in":[25,197],"distributed":[26,76],"serving":[27],"systems":[28],"arises":[29],"from":[30],"redundant":[32],"processing":[33],"requests":[35,84,107],"that":[36],"share":[37],"common":[38],"prefixes,":[39,110],"such":[40],"as":[41,206],"system":[42,72,112,159],"prompts":[43],"or":[44],"few-shot":[45],"examples.":[46],"Traditional":[47],"contentagnostic":[48],"load":[49],"balancers":[50],"fail":[51],"to":[52,57,74,85,179],"exploit":[53],"these":[54],"redundancies,":[55],"leading":[56],"inefficient":[58],"resource":[59],"utilization":[60],"increased":[62,160],"latency.":[63],"This":[64],"paper":[65],"introduces":[66],"a":[67,139,145,180,188,207],"dynamic,":[68],"prefix-aware":[69],"request":[70],"scheduling":[71],"designed":[73],"optimize":[75],"LLM":[77,221],"serving.":[78],"Our":[79],"approach":[80,205],"intelligently":[81],"routes":[82],"incoming":[83],"specific":[86],"GPU":[87,142],"workers":[88],"analyzing":[90],"prompt":[91],"content":[92],"matching":[94],"it":[95],"with":[96,108],"resident":[98],"Key-Value":[99],"(KV)":[100],"caches":[101],"across":[102],"cluster.":[104],"By":[105],"colocating":[106],"shared":[109],"our":[111,158,204],"maximizes":[113],"KV":[114],"cache":[115,200],"reuse,":[116],"minimizes":[117],"expensive":[118],"prefill":[119,163],"computations,":[120],"enables":[122],"more":[123],"efficient":[124],"batched":[125],"attention":[126],"operations":[127],"at":[128],"worker":[130],"level.":[131],"We":[132],"implemented":[133],"evaluated":[135],"this":[136],"scheduler":[137],"on":[138],"12":[140],"-node":[141],"cluster":[143],"using":[144],"real-world":[146],"chatbot":[147],"workload.":[148],"results":[150],"demonstrate":[151],"profound":[153],"impact":[154],"content-aware":[156],"scheduling:":[157],"aggregate":[162],"throughput":[164,216],"over":[166,175],"144":[167],"%":[168,177,194],"reduced":[170],"median":[172],"Time-to-First-Token":[173],"40":[176],"compared":[178],"conventional":[181],"Round-Robin":[182],"policy.":[183],"These":[184],"performance":[185],"gains":[186],"are":[187],"direct":[189],"result":[190],"an":[192],"82":[193],"relative":[195],"increase":[196],"prefix":[199],"hit":[201],"rate,":[202],"validating":[203],"highly":[208],"effective":[209],"cost-efficient":[211],"strategy":[212],"for":[213],"enhancing":[214],"responsiveness":[218],"large-scale":[220],"services.":[222]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-01-15T00:00:00"}