{"id":"https://openalex.org/W7153128335","doi":"https://doi.org/10.48550/arxiv.2604.08075","title":"Dual-Pool Token-Budget Routing for Cost-Efficient and Reliable LLM Serving","display_name":"Dual-Pool Token-Budget Routing for Cost-Efficient and Reliable LLM Serving","publication_year":2026,"publication_date":"2026-04-09","ids":{"openalex":"https://openalex.org/W7153128335","doi":"https://doi.org/10.48550/arxiv.2604.08075"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.08075","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.08075","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.08075","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133322111","display_name":"Xunzhuo Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xunzhuo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130113881","display_name":"Bowei He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Bowei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133393917","display_name":"Xue Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133366537","display_name":"Andy Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Andy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133371233","display_name":"Haichen Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Haichen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133365370","display_name":"Huamin Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Huamin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.3434000015258789,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.3434000015258789,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.2809000015258789,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.09870000183582306,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6288999915122986},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.611299991607666},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.6067000031471252},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.59579998254776},{"id":"https://openalex.org/keywords/downtime","display_name":"Downtime","score":0.4627000093460083},{"id":"https://openalex.org/keywords/preemption","display_name":"Preemption","score":0.4521999955177307},{"id":"https://openalex.org/keywords/routing","display_name":"Routing (electronic design automation)","score":0.43140000104904175},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.37959998846054077},{"id":"https://openalex.org/keywords/homogeneous","display_name":"Homogeneous","score":0.36579999327659607}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7231000065803528},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6288999915122986},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.611299991607666},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.6067000031471252},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.59579998254776},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.5300999879837036},{"id":"https://openalex.org/C180591934","wikidata":"https://www.wikidata.org/wiki/Q1253369","display_name":"Downtime","level":2,"score":0.4627000093460083},{"id":"https://openalex.org/C206952183","wikidata":"https://www.wikidata.org/wiki/Q1193100","display_name":"Preemption","level":2,"score":0.4521999955177307},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.43140000104904175},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.37959998846054077},{"id":"https://openalex.org/C66882249","wikidata":"https://www.wikidata.org/wiki/Q169336","display_name":"Homogeneous","level":2,"score":0.36579999327659607},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.35510000586509705},{"id":"https://openalex.org/C2781007418","wikidata":"https://www.wikidata.org/wiki/Q1456934","display_name":"Capacity planning","level":2,"score":0.3522999882698059},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.3483000099658966},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.3481000065803528},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.34119999408721924},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.33980000019073486},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.334199994802475},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.3334999978542328},{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.33329999446868896},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.33230000734329224},{"id":"https://openalex.org/C2778348673","wikidata":"https://www.wikidata.org/wiki/Q739302","display_name":"Production (economics)","level":2,"score":0.3310999870300293},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.3165999948978424},{"id":"https://openalex.org/C2780945871","wikidata":"https://www.wikidata.org/wiki/Q194274","display_name":"Backup","level":2,"score":0.3000999987125397},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.29330000281333923},{"id":"https://openalex.org/C8505890","wikidata":"https://www.wikidata.org/wiki/Q605095","display_name":"Budget constraint","level":2,"score":0.27140000462532043},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2651999890804291},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.263700008392334},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.26330000162124634},{"id":"https://openalex.org/C55350006","wikidata":"https://www.wikidata.org/wiki/Q237193","display_name":"Exponential distribution","level":2,"score":0.25619998574256897}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.08075","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.08075","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.08075","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.08075","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.458988219499588,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Production":[0],"vLLM":[1],"fleets":[2],"typically":[3],"provision":[4],"each":[5],"instance":[6],"for":[7,33,58,121],"the":[8,119,156],"worst-case":[9],"context":[10],"length,":[11],"leading":[12],"to":[13,145,149,177,221],"substantial":[14],"KV-cache":[15],"over-allocation":[16],"and":[17,40,49,85,139,161,190,224,235],"under-utilized":[18],"concurrency.":[19],"In":[20],"practice,":[21],"80-95%":[22],"of":[23],"requests":[24],"are":[25,28],"short,":[26],"yet":[27],"served":[29],"under":[30],"configurations":[31],"optimized":[32],"long":[34],"contexts,":[35],"wasting":[36],"4-8$\\times$":[37],"throughput":[38,141],"capacity":[39],"triggering":[41],"reliability":[42],"issues":[43],"such":[44,230],"as":[45,231],"OOM":[46],"crashes,":[47],"preemption,":[48],"request":[50,91],"rejections.":[51],"We":[52,63,124],"identify":[53],"a":[54,68,74,81,86,103,122,127],"common":[55],"root":[56],"cause":[57],"these":[59],"inefficiencies:":[60],"configuration-traffic":[61],"mismatch.":[62],"propose":[64],"dual-pool":[65],"token-budget":[66],"routing,":[67],"lightweight":[69],"dispatch":[70,217],"mechanism":[71],"that":[72,107,131,169],"partitions":[73],"homogeneous":[75],"fleet":[76,182],"into":[77],"two":[78],"specialized":[79],"pools:":[80],"high-throughput":[82],"short-context":[83],"pool":[84],"high-capacity":[87],"long-context":[88],"pool.":[89],"Each":[90],"is":[92,108],"routed":[93],"based":[94],"on":[95,152,165,201],"its":[96],"estimated":[97],"total":[98],"token":[99],"budget,":[100],"computed":[101],"using":[102],"per-category":[104],"bytes-to-token":[105],"ratio":[106],"learned":[109],"online":[110],"via":[111],"exponential":[112],"moving":[113],"average":[114],"from":[115,136,155],"usage.prompt_tokens":[116],"feedback,":[117],"eliminating":[118],"need":[120],"tokenizer.":[123],"also":[125],"develop":[126],"simple":[128],"analytical":[129],"model":[130],"predicts":[132],"fleet-level":[133],"cost":[134],"savings":[135,180],"workload":[137],"characteristics":[138],"measured":[140],"differences,":[142],"enabling":[143],"practitioners":[144],"estimate":[146],"benefits":[147],"prior":[148],"deployment.":[150],"Evaluations":[151],"real-world":[153],"traces":[154],"Azure":[157],"LLM":[158],"Inference":[159],"Dataset":[160],"LMSYS-Chat-1M,":[162],"serving":[163],"Llama-3-70B":[164],"A100":[166],"GPUs,":[167],"show":[168],"our":[170],"approach":[171],"reduces":[172],"GPU-hours":[173],"by":[174,188,194],"31-42%,":[175],"corresponding":[176],"\\$2.86M":[178],"annual":[179,210],"at":[181,204],"scale,":[183],"while":[184],"lowering":[185],"preemption":[186],"rates":[187],"5.4$\\times$":[189],"improving":[191],"P99":[192],"TTFT":[193],"6%.":[195],"A":[196],"case":[197],"study":[198],"with":[199,227],"Qwen3-235B-A22B":[200],"AMD":[202],"MI300X":[203],"10,000":[205],"req/s":[206],"projects":[207],"\\$15.4M":[208],"in":[209],"savings.":[211],"The":[212],"method":[213],"incurs":[214],"only":[215],"O(1)":[216],"overhead,":[218],"adapts":[219],"automatically":[220],"heterogeneous":[222],"workloads,":[223],"composes":[225],"seamlessly":[226],"existing":[228],"optimizations":[229],"PagedAttention,":[232],"continuous":[233],"batching,":[234],"prefill-decode":[236],"disaggregation.":[237]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-11T00:00:00"}