{"id":"https://openalex.org/W7165789940","doi":"https://doi.org/10.48550/arxiv.2606.24506","title":"CrossPool: Efficient Multi-LLM Serving for Cold MoE Models through KV-Cache and Weight Disaggregation","display_name":"CrossPool: Efficient Multi-LLM Serving for Cold MoE Models through KV-Cache and Weight Disaggregation","publication_year":2026,"publication_date":"2026-06-23","ids":{"openalex":"https://openalex.org/W7165789940","doi":"https://doi.org/10.48550/arxiv.2606.24506"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.24506","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.24506","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.24506","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5021782599","display_name":"Z Zhenyu Ye","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ye, Zhuoren","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066123431","display_name":"Tianyu Wo","orcid":"https://orcid.org/0000-0002-5331-3364"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wo, Tianyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111276571","display_name":"Dinghao Xue","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xue, Dinghao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139285802","display_name":"Mingming Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Mingming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139258094","display_name":"Yuchen Teng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Teng, Yuchen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139255651","display_name":"Chunming Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Chunming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139295069","display_name":"Renyu Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Renyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.5282999873161316,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.5282999873161316,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.16689999401569366,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.12630000710487366,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.5712000131607056},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.4593999981880188},{"id":"https://openalex.org/keywords/dynamic-random-access-memory","display_name":"Dynamic random-access memory","score":0.45179998874664307},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.4255000054836273},{"id":"https://openalex.org/keywords/transient","display_name":"Transient (computer programming)","score":0.4174000024795532},{"id":"https://openalex.org/keywords/host","display_name":"Host (biology)","score":0.38260000944137573},{"id":"https://openalex.org/keywords/memory-management","display_name":"Memory management","score":0.32249999046325684}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6769000291824341},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.5712000131607056},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.4593999981880188},{"id":"https://openalex.org/C118702147","wikidata":"https://www.wikidata.org/wiki/Q189396","display_name":"Dynamic random-access memory","level":3,"score":0.45179998874664307},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.4255000054836273},{"id":"https://openalex.org/C2780799671","wikidata":"https://www.wikidata.org/wiki/Q17087362","display_name":"Transient (computer programming)","level":2,"score":0.4174000024795532},{"id":"https://openalex.org/C126831891","wikidata":"https://www.wikidata.org/wiki/Q221673","display_name":"Host (biology)","level":2,"score":0.38260000944137573},{"id":"https://openalex.org/C176649486","wikidata":"https://www.wikidata.org/wiki/Q2308807","display_name":"Memory management","level":3,"score":0.32249999046325684},{"id":"https://openalex.org/C20556612","wikidata":"https://www.wikidata.org/wiki/Q4469374","display_name":"Volume (thermodynamics)","level":2,"score":0.3154999911785126},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.3091000020503998},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.29120001196861267},{"id":"https://openalex.org/C149629883","wikidata":"https://www.wikidata.org/wiki/Q660926","display_name":"Fraction (chemistry)","level":2,"score":0.2775000035762787},{"id":"https://openalex.org/C2778956030","wikidata":"https://www.wikidata.org/wiki/Q5142477","display_name":"Cold start (automotive)","level":2,"score":0.2768000066280365},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.2685999870300293},{"id":"https://openalex.org/C2777762679","wikidata":"https://www.wikidata.org/wiki/Q2352600","display_name":"Food contact materials","level":3,"score":0.2685000002384186},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.25999999046325684},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.25699999928474426}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.24506","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.24506","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.24506","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.24506","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Emerging":[0],"LLM":[1],"services":[2],"increasingly":[3],"host":[4],"many":[5],"sparse":[6,13],"MoE":[7,122],"models,":[8,144],"yet":[9],"most":[10],"models":[11,38,123],"receive":[12],"requests":[14,153,195],"and":[15,28,34,74,89,110,128,145,165,175,196],"remain":[16,76],"cold.":[17],"This":[18],"creates":[19],"a":[20,56,78,98,117,135,146,162,167],"GPU":[21,80,107,132,188],"memory":[22,81,108,133,189],"problem:":[23],"model":[24,53],"weights":[25,73,84,127,136,141],"are":[26],"stable":[27],"model-determined,":[29],"while":[30,154],"KV-cache":[31,42,58,67,75,129,147,163],"is":[32,69],"transient":[33],"demand-determined.":[35],"Because":[36],"cold":[37,121,143],"rarely":[39],"reach":[40],"peak":[41],"demand":[43],"at":[44],"the":[45,198],"same":[46],"time,":[47],"reserving":[48],"worst-case":[49],"KV":[50,102],"capacity":[51],"per":[52],"wastes":[54],"memory;":[55],"shared":[57],"pool":[59,137,148],"can":[60],"instead":[61],"provision":[62],"aggregate":[63],"active":[64,152],"demand.":[65],"However,":[66],"sharing":[68],"not":[70],"sufficient":[71],"when":[72],"in":[77],"monolithic":[79],"pool.":[82],"Static":[83],"compete":[85],"with":[86,178],"dynamic":[87],"KV-cache,":[88],"KV-head-limited":[90],"attention":[91,156],"under":[92],"cold,":[93],"low-concurrency":[94],"traffic":[95],"exposes":[96],"only":[97],"fraction":[99],"of":[100],"replicated":[101],"capacity,":[103],"leading":[104],"to":[105,158,181,209],"low":[106],"utilization":[109],"weak":[111],"long-context":[112,194],"support.":[113],"We":[114],"present":[115],"CrossPool,":[116],"serving":[118,202],"engine":[119],"for":[120],"that":[124,138,149,171],"separates":[125],"FFN":[126,140],"into":[130],"two":[131],"pools:":[134],"consolidates":[139],"across":[142],"dynamically":[150],"serves":[151],"keeping":[155],"local":[157],"KV-cache.":[159],"CrossPool":[160,191],"combines":[161],"planner":[164],"virtualizer,":[166],"layer-wise":[168],"pipeline":[169],"scheduler":[170],"hides":[172],"hidden-state":[173],"transfers,":[174],"persistent":[176],"kernels":[177],"control":[179,184],"lowering":[180],"reduce":[182],"CPU-GPU":[183],"overhead.":[185],"With":[186],"efficient":[187],"pooling,":[190],"underpins":[192],"bursty":[193],"outperforms":[197],"state-of-the-art":[199],"kvcached-based":[200],"multi-LLM":[201],"system,":[203],"reducing":[204],"P99":[205],"TBT":[206],"by":[207],"up":[208],"10.4x.":[210]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-25T00:00:00"}