{"id":"https://openalex.org/W4415275341","doi":"https://doi.org/10.1002/spe.70054","title":"BanaServe: Unified KV Cache and Dynamic Module Migration for Balancing Disaggregated LLM Serving in AI Infrastructure","display_name":"BanaServe: Unified KV Cache and Dynamic Module Migration for Balancing Disaggregated LLM Serving in AI Infrastructure","publication_year":2026,"publication_date":"2026-01-21","ids":{"openalex":"https://openalex.org/W4415275341","doi":"https://doi.org/10.1002/spe.70054"},"language":"en","primary_location":{"id":"doi:10.1002/spe.70054","is_oa":false,"landing_page_url":"https://doi.org/10.1002/spe.70054","pdf_url":null,"source":{"id":"https://openalex.org/S122199241","display_name":"Software Practice and Experience","issn_l":"0038-0644","issn":["0038-0644","1097-024X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Software: Practice and Experience","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2510.13223","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045331954","display_name":"Yan\u2010Lin He","orcid":"https://orcid.org/0000-0001-9999-3679"},"institutions":[{"id":"https://openalex.org/I3045169105","display_name":"Southern University of Science and Technology","ror":"https://ror.org/049tv2d57","country_code":"CN","type":"education","lineage":["https://openalex.org/I3045169105"]},{"id":"https://openalex.org/I4210145761","display_name":"Shenzhen Institutes of Advanced Technology","ror":"https://ror.org/04gh4er46","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210145761"]},{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yiyuan He","raw_affiliation_strings":["AIOS Team Alibaba Group Inc.  HangZhou China","Shenzhen Institutes of Advanced Technology Chinese Academy of Sciences  Shenzhen China","Southern University of Science and Technology  Shenzhen China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AIOS Team Alibaba Group Inc.  HangZhou China","institution_ids":["https://openalex.org/I45928872"]},{"raw_affiliation_string":"Shenzhen Institutes of Advanced Technology Chinese Academy of Sciences  Shenzhen China","institution_ids":["https://openalex.org/I4210145761"]},{"raw_affiliation_string":"Southern University of Science and Technology  Shenzhen China","institution_ids":["https://openalex.org/I3045169105"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014474110","display_name":"Minxian Xu","orcid":"https://orcid.org/0000-0002-0046-5153"},"institutions":[{"id":"https://openalex.org/I4210145761","display_name":"Shenzhen Institutes of Advanced Technology","ror":"https://ror.org/04gh4er46","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210145761"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Minxian Xu","raw_affiliation_strings":["Shenzhen Institutes of Advanced Technology Chinese Academy of Sciences  Shenzhen China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shenzhen Institutes of Advanced Technology Chinese Academy of Sciences  Shenzhen China","institution_ids":["https://openalex.org/I4210145761"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102796060","display_name":"Jingfeng Wu","orcid":"https://orcid.org/0009-0009-3414-4487"},"institutions":[{"id":"https://openalex.org/I4210145761","display_name":"Shenzhen Institutes of Advanced Technology","ror":"https://ror.org/04gh4er46","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210145761"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jingfeng Wu","raw_affiliation_strings":["Shenzhen Institutes of Advanced Technology Chinese Academy of Sciences  Shenzhen China","University of Chinese Academy of Sciences  China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shenzhen Institutes of Advanced Technology Chinese Academy of Sciences  Shenzhen China","institution_ids":["https://openalex.org/I4210145761"]},{"raw_affiliation_string":"University of Chinese Academy of Sciences  China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111985787","display_name":"Juejun Hu","orcid":"https://orcid.org/0000-0002-7233-3918"},"institutions":[{"id":"https://openalex.org/I3045169105","display_name":"Southern University of Science and Technology","ror":"https://ror.org/049tv2d57","country_code":"CN","type":"education","lineage":["https://openalex.org/I3045169105"]},{"id":"https://openalex.org/I4210145761","display_name":"Shenzhen Institutes of Advanced Technology","ror":"https://ror.org/04gh4er46","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210145761"]},{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianmin Hu","raw_affiliation_strings":["AIOS Team Alibaba Group Inc.  HangZhou China","Shenzhen Institutes of Advanced Technology Chinese Academy of Sciences  Shenzhen China","Southern University of Science and Technology  Shenzhen China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AIOS Team Alibaba Group Inc.  HangZhou China","institution_ids":["https://openalex.org/I45928872"]},{"raw_affiliation_string":"Shenzhen Institutes of Advanced Technology Chinese Academy of Sciences  Shenzhen China","institution_ids":["https://openalex.org/I4210145761"]},{"raw_affiliation_string":"Southern University of Science and Technology  Shenzhen China","institution_ids":["https://openalex.org/I3045169105"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055979923","display_name":"Chong Ma","orcid":"https://orcid.org/0000-0002-5423-4449"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chong Ma","raw_affiliation_strings":["AIOS Team Alibaba Group Inc.  HangZhou China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AIOS Team Alibaba Group Inc.  HangZhou China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028342450","display_name":"Min Shen","orcid":"https://orcid.org/0000-0002-3510-5705"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Min Shen","raw_affiliation_strings":["AIOS Team Alibaba Group Inc.  HangZhou China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AIOS Team Alibaba Group Inc.  HangZhou China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100420755","display_name":"Le Chen","orcid":"https://orcid.org/0000-0003-3789-2578"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Le Chen","raw_affiliation_strings":["AIOS Team Alibaba Group Inc.  HangZhou China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AIOS Team Alibaba Group Inc.  HangZhou China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100569901","display_name":"Chengzhong Xu","orcid":null},"institutions":[{"id":"https://openalex.org/I204512498","display_name":"University of Macau","ror":"https://ror.org/01r4q9n85","country_code":"MO","type":"education","lineage":["https://openalex.org/I204512498"]}],"countries":["MO"],"is_corresponding":false,"raw_author_name":"Chengzhong Xu","raw_affiliation_strings":["State Key Lab of IOTSC, Faculty of Science and Technology University of Macau  Macau SAR China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Lab of IOTSC, Faculty of Science and Technology University of Macau  Macau SAR China","institution_ids":["https://openalex.org/I204512498"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100763937","display_name":"Qu Lin","orcid":"https://orcid.org/0000-0002-4251-5577"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lin Qu","raw_affiliation_strings":["AIOS Team Alibaba Group Inc.  HangZhou China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AIOS Team Alibaba Group Inc.  HangZhou China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5009982405","display_name":"YE Ke-jiang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210145761","display_name":"Shenzhen Institutes of Advanced Technology","ror":"https://ror.org/04gh4er46","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210145761"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kejiang Ye","raw_affiliation_strings":["Shenzhen Institutes of Advanced Technology Chinese Academy of Sciences  Shenzhen China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shenzhen Institutes of Advanced Technology Chinese Academy of Sciences  Shenzhen China","institution_ids":["https://openalex.org/I4210145761"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5014474110"],"corresponding_institution_ids":["https://openalex.org/I4210145761"],"apc_list":{"value":4220,"currency":"USD","value_usd":4220},"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.00705882,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"56","issue":"4","first_page":"424","last_page":"444"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12326","display_name":"Network Packet Processing and Optimization","score":0.9386000037193298,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12326","display_name":"Network Packet Processing and Optimization","score":0.9386000037193298,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9358999729156494,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.935699999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.779699981212616},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.5023000240325928},{"id":"https://openalex.org/keywords/cache-invalidation","display_name":"Cache invalidation","score":0.3977999985218048},{"id":"https://openalex.org/keywords/cache-algorithms","display_name":"Cache algorithms","score":0.3939000070095062},{"id":"https://openalex.org/keywords/shared-resource","display_name":"Shared resource","score":0.36500000953674316},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.35920000076293945},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.3474000096321106},{"id":"https://openalex.org/keywords/load-balancing","display_name":"Load balancing (electrical power)","score":0.33809998631477356}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8519999980926514},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.779699981212616},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.5909000039100647},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.5023000240325928},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4968999922275543},{"id":"https://openalex.org/C25536678","wikidata":"https://www.wikidata.org/wiki/Q5015977","display_name":"Cache invalidation","level":5,"score":0.3977999985218048},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.3939000070095062},{"id":"https://openalex.org/C51332947","wikidata":"https://www.wikidata.org/wiki/Q1172305","display_name":"Shared resource","level":2,"score":0.36500000953674316},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.35920000076293945},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.3474000096321106},{"id":"https://openalex.org/C138959212","wikidata":"https://www.wikidata.org/wiki/Q1806783","display_name":"Load balancing (electrical power)","level":3,"score":0.33809998631477356},{"id":"https://openalex.org/C201148951","wikidata":"https://www.wikidata.org/wiki/Q5015976","display_name":"Cache coloring","level":4,"score":0.3334999978542328},{"id":"https://openalex.org/C3720319","wikidata":"https://www.wikidata.org/wiki/Q5015937","display_name":"Cache-only memory architecture","level":5,"score":0.3172000050544739},{"id":"https://openalex.org/C167713795","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"Smart Cache","level":5,"score":0.31150001287460327},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.30799999833106995},{"id":"https://openalex.org/C36351404","wikidata":"https://www.wikidata.org/wiki/Q7247295","display_name":"Process migration","level":3,"score":0.28209999203681946},{"id":"https://openalex.org/C113166858","wikidata":"https://www.wikidata.org/wiki/Q5015981","display_name":"Cache pollution","level":5,"score":0.28029999136924744},{"id":"https://openalex.org/C29202148","wikidata":"https://www.wikidata.org/wiki/Q287260","display_name":"Resource allocation","level":2,"score":0.2793999910354614},{"id":"https://openalex.org/C88196245","wikidata":"https://www.wikidata.org/wiki/Q8034984","display_name":"Working set","level":2,"score":0.265500009059906},{"id":"https://openalex.org/C2777958785","wikidata":"https://www.wikidata.org/wiki/Q17120940","display_name":"Resource efficiency","level":2,"score":0.26080000400543213},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.251800000667572}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1002/spe.70054","is_oa":false,"landing_page_url":"https://doi.org/10.1002/spe.70054","pdf_url":null,"source":{"id":"https://openalex.org/S122199241","display_name":"Software Practice and Experience","issn_l":"0038-0644","issn":["0038-0644","1097-024X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Software: Practice and Experience","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2510.13223","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.13223","pdf_url":"https://arxiv.org/pdf/2510.13223","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2510.13223","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.13223","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2510.13223","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.13223","pdf_url":"https://arxiv.org/pdf/2510.13223","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3125402525","display_name":null,"funder_award_id":"2024A1515010251","funder_id":"https://openalex.org/F4320337111","funder_display_name":"Basic and Applied Basic Research Foundation of Guangdong Province"},{"id":"https://openalex.org/G5840464552","display_name":null,"funder_award_id":"2023B1515130002","funder_id":"https://openalex.org/F4320337111","funder_display_name":"Basic and Applied Basic Research Foundation of Guangdong Province"},{"id":"https://openalex.org/G6117987806","display_name":null,"funder_award_id":"62572462","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320337111","display_name":"Basic and Applied Basic Research Foundation of Guangdong Province","ror":null}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415275341.pdf"},"referenced_works_count":10,"referenced_works":["https://openalex.org/W4410214245","https://openalex.org/W4412899734","https://openalex.org/W4409108393","https://openalex.org/W4401211704","https://openalex.org/W4387321091","https://openalex.org/W2979826702","https://openalex.org/W4404401017","https://openalex.org/W4409963643","https://openalex.org/W4411486537","https://openalex.org/W4394998727"],"related_works":[],"abstract_inverted_index":{"ABSTRACT":[0],"Objective":[1],"Large":[2],"Language":[3],"Models":[4],"(LLMs)":[5],"are":[6,267,291],"increasingly":[7],"deployed":[8],"in":[9,317],"modern":[10],"AI":[11],"infrastructure,":[12],"creating":[13],"a":[14,44,116,165,323],"strong":[15],"demand":[16],"for":[17,120,158,293,328],"high\u2010throughput":[18],"and":[19,38,74,78,85,96,129,134,163,187,211,219,228,244,288,305,314,325,336],"resource\u2010efficient":[20],"serving":[21,50,123,207,278,331],"systems.":[22],"Disaggregated":[23],"LLM":[24,122,206,296,330],"serving,":[25],"which":[26],"decouples":[27],"prompt":[28],"prefill":[29,73,133],"from":[30,53,178],"auto\u2010regressive":[31],"decode":[32,76,135],"to":[33,62,64,99,146,174,190,237],"accommodate":[34],"their":[35],"heterogeneous":[36,337],"compute":[37],"memory":[39,130,160],"characteristics,":[40],"has":[41],"emerged":[42],"as":[43],"promising":[45],"architecture.":[46],"However,":[47],"existing":[48],"disaggregated":[49,121,295],"systems":[51,332],"suffer":[52],"three":[54,139],"fundamental":[55],"limitations:":[56],"static":[57],"resource":[58,93,286,301,315],"allocation":[59],"that":[60,81,124,283],"fails":[61],"adapt":[63],"highly":[65,276],"dynamic":[66,117,277,335],"workloads,":[67,221],"severe":[68],"load":[69,83,161,307],"imbalance":[70],"between":[71],"compute\u2010bound":[72],"memory\u2010bound":[75],"stages,":[77],"prefix\u2010cache\u2010aware":[79],"routing":[80,176,290],"skews":[82],"distribution":[84],"creates":[86],"performance":[87],"hotspots.":[88],"These":[89,265],"issues":[90],"collectively":[91],"limit":[92],"utilization,":[94],"scalability,":[95],"the":[97],"ability":[98],"meet":[100],"service":[101],"level":[102],"objectives":[103],"(SLOs)":[104],"under":[105,217,275,334],"real\u2010world":[106,318],"workloads.":[107,338],"Methods":[108],"To":[109],"address":[110],"these":[111,182],"challenges,":[112],"we":[113],"propose":[114],"BanaServe,":[115],"orchestration":[118],"framework":[119],"continuously":[125],"rebalances":[126],"both":[127],"computational":[128],"resources":[131],"across":[132,269],"instances.":[136],"BanaServe":[137,199,216,239,255,281,309],"introduces":[138],"key":[140],"mechanisms:":[141],"(i)":[142],"layer\u2010level":[143],"weight":[144],"migration":[145,157],"enable":[147],"coarse\u2010grained":[148],"redistribution":[149],"of":[150,204],"computation,":[151],"(ii)":[152],"attention\u2010level":[153],"Key\u2013Value":[154],"(KV)":[155],"cache":[156,179],"fine\u2010grained":[159],"balancing,":[162],"(iii)":[164],"Global":[166],"KV":[167],"Cache":[168],"Store":[169],"with":[170,195,253],"layer\u2010wise":[171],"overlapped":[172],"transmission":[173],"decouple":[175],"decisions":[177],"placement.":[180],"Together,":[181],"mechanisms":[183],"eliminate":[184],"cache\u2010induced":[185,306],"hotspots":[186],"allow":[188],"routers":[189],"perform":[191],"purely":[192],"load\u2010aware":[193],"scheduling":[194],"minimal":[196],"latency":[197,262],"overhead.":[198],"is":[200],"implemented":[201],"on":[202],"top":[203],"state\u2010of\u2010the\u2010art":[205],"frameworks,":[208],"including":[209,222],"vLLM":[210],"DistServe.":[212],"Results":[213],"We":[214],"evaluate":[215],"diverse":[218],"challenging":[220],"long\u2010context":[223],"inference,":[224],"bursty":[225],"request":[226],"arrivals,":[227],"mixed":[229],"prompt\u2013generation":[230],"patterns.":[231],"Experimental":[232],"results":[233],"show":[234],"that,":[235],"compared":[236],"vLLM,":[238],"improves":[240,311],"throughput":[241,259],"by":[242,249,263],"1.2\u20133.9\u00d7":[243],"reduces":[245],"total":[246],"processing":[247],"time":[248],"3.9%\u201378.4%.":[250],"In":[251],"comparison":[252],"DistServe,":[254],"achieves":[256],"1.1\u20132.8\u00d7":[257],"higher":[258],"while":[260],"reducing":[261],"1.4%\u201370.1%.":[264],"gains":[266],"consistent":[268],"workload":[270],"variations,":[271],"demonstrating":[272],"BanaServe's":[273],"robustness":[274],"conditions.":[279],"Conclusion":[280],"demonstrates":[282],"dynamic,":[284],"multi\u2010granularity":[285],"rebalancing":[287],"cache\u2010decoupled":[289],"essential":[292],"efficient":[294],"serving.":[297],"By":[298],"jointly":[299],"addressing":[300],"elasticity,":[302],"stage":[303],"imbalance,":[304],"skew,":[308],"substantially":[310],"throughput,":[312],"latency,":[313],"utilization":[316],"deployments.":[319],"This":[320],"work":[321],"provides":[322],"practical":[324],"scalable":[326],"foundation":[327],"next\u2010generation":[329],"operating":[333]},"counts_by_year":[],"updated_date":"2026-08-10T07:06:39.071514","created_date":"2025-10-17T00:00:00"}