{"id":"https://openalex.org/W7224954337","doi":"https://doi.org/10.48550/arxiv.2610.40292","title":"DolphinRouting: Cluster-Level Binary Routing for Disaggregated LLM Inference Serving","display_name":"DolphinRouting: Cluster-Level Binary Routing for Disaggregated LLM Inference Serving","publication_year":2026,"publication_date":"2026-10-29","ids":{"openalex":"https://openalex.org/W7224954337","doi":"https://doi.org/10.48550/arxiv.2610.40292"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.40292","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.40292","pdf_url":"https://arxiv.org/pdf/2610.40292","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.40292","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5160214229","display_name":"Yishuo Yuan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuan, Yishuo","raw_affiliation_strings":["Nanjing University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanjing University","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102816452","display_name":"Shaoyu Yang","orcid":"https://orcid.org/0009-0008-7651-8582","observed_orcids":["https://orcid.org/0009-0008-7651-8582"]},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yang, Shaoyu","raw_affiliation_strings":["Nanjing University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanjing University","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075174750","display_name":"Chunrong Fang","orcid":"https://orcid.org/0000-0002-9930-7111","observed_orcids":["https://orcid.org/0000-0002-9930-7111"]},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fang, Chunrong","raw_affiliation_strings":["Nanjing University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanjing University","institution_ids":["https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I881766915"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.7045999765396118,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.7045999765396118,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.07100000232458115,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.05180000141263008,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-model-inference","display_name":"large language model inference","score":0.722000002861023},{"id":"https://openalex.org/keywords/request-routing","display_name":"request routing","score":0.7720000147819519},{"id":"https://openalex.org/keywords/service-level-objectives","display_name":"service level objectives","score":0.5830000042915344},{"id":"https://openalex.org/keywords/cache-reuse","display_name":"cache reuse","score":0.4950000047683716},{"id":"https://openalex.org/keywords/k-means-clustering","display_name":"K-means clustering","score":0.7160000205039978},{"id":"https://openalex.org/keywords/online-feature-extraction","display_name":"online feature extraction","score":0.7319999933242798},{"id":"https://openalex.org/keywords/latency-optimization","display_name":"latency optimization","score":0.4309999942779541},{"id":"https://openalex.org/keywords/throughput-optimization","display_name":"throughput maximization","score":0.6100000143051147},{"id":"https://openalex.org/keywords/mixed-workloads","display_name":"mixed workloads","score":0.4819999933242798}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7926999926567078},{"id":"https://openalex.org/C45235069","wikidata":"https://www.wikidata.org/wiki/Q278425","display_name":"Table (database)","level":2,"score":0.5475999712944031},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.5475000143051147},{"id":"https://openalex.org/C164866538","wikidata":"https://www.wikidata.org/wiki/Q367351","display_name":"Cluster (spacecraft)","level":2,"score":0.5098000168800354},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.5048999786376953},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.47440001368522644},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.4596000015735626},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.4530999958515167},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.42570000886917114},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4221000075340271},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.4205999970436096},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4185999929904938},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.3970000147819519},{"id":"https://openalex.org/C91154448","wikidata":"https://www.wikidata.org/wiki/Q623818","display_name":"Binary search tree","level":3,"score":0.35679998993873596},{"id":"https://openalex.org/C2779190172","wikidata":"https://www.wikidata.org/wiki/Q4913888","display_name":"Binary data","level":3,"score":0.3400999903678894},{"id":"https://openalex.org/C148483581","wikidata":"https://www.wikidata.org/wiki/Q446488","display_name":"Feature selection","level":2,"score":0.3310999870300293},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3276999890804291},{"id":"https://openalex.org/C71559656","wikidata":"https://www.wikidata.org/wiki/Q671298","display_name":"Divide and conquer algorithms","level":2,"score":0.32519999146461487},{"id":"https://openalex.org/C43364308","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Byte","level":2,"score":0.3212999999523163},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.3183000087738037},{"id":"https://openalex.org/C134835016","wikidata":"https://www.wikidata.org/wiki/Q690265","display_name":"Lookup table","level":2,"score":0.3070000112056732},{"id":"https://openalex.org/C184896649","wikidata":"https://www.wikidata.org/wiki/Q290066","display_name":"Routing table","level":4,"score":0.30559998750686646},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.29989999532699585},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.29980000853538513},{"id":"https://openalex.org/C59656382","wikidata":"https://www.wikidata.org/wiki/Q191536","display_name":"Conjunction (astronomy)","level":2,"score":0.29899999499320984},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.2831999957561493},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.27790001034736633},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.2718999981880188},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2653999924659729},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.26499998569488525},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.2623000144958496},{"id":"https://openalex.org/C29202148","wikidata":"https://www.wikidata.org/wiki/Q287260","display_name":"Resource allocation","level":2,"score":0.26190000772476196},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.2531999945640564}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.40292","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.40292","pdf_url":"https://arxiv.org/pdf/2610.40292","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.40292","doi":"https://doi.org/10.48550/arxiv.2610.40292","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.40292","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.40292","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.40292","pdf_url":"https://arxiv.org/pdf/2610.40292","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7224954337.pdf","grobid_xml":"https://content.openalex.org/works/W7224954337.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7138268496","https://openalex.org/W7213397274","https://openalex.org/W4391045969","https://openalex.org/W7133358891","https://openalex.org/W7167278146","https://openalex.org/W7214203319","https://openalex.org/W7134032903","https://openalex.org/W4417101736","https://openalex.org/W7167030261","https://openalex.org/W4406023746"],"abstract_inverted_index":{"Prefill-decode":[0],"(PD)":[1],"disaggregation":[2],"isolates":[3],"the":[4,101],"two":[5],"phases":[6],"of":[7],"large":[8],"language":[9],"model":[10],"serving,":[11],"but":[12],"request-level":[13],"path":[14],"selection":[15],"must":[16],"account":[17],"for":[18],"prefill":[19],"work,":[20],"cache":[21],"reuse,":[22],"and":[23,57,74,97,104,115],"decode":[24],"demand.":[25],"We":[26,82],"present":[27],"DolphinRouting,":[28],"which":[29],"learns":[30,63],"this":[31],"choice":[32],"from":[33],"end-to-end":[34],"SLO":[35],"feedback":[36],"under":[37],"a":[38,86,94],"given":[39],"P/D":[40],"resource":[41],"allocation.":[42],"Requests":[43],"are":[44],"clustered":[45],"by":[46,113,118],"raw":[47],"prompt":[48],"length,":[49],"cache-adjusted":[50],"input":[51],"lengths":[52],"on":[53,85],"both":[54],"worker":[55],"pools,":[56],"output-length":[58],"information.":[59],"Offline":[60],"policy":[61],"search":[62],"one":[64],"binary":[65],"action":[66],"per":[67],"cluster.":[68],"Online":[69],"feature":[70],"extraction,":[71],"cluster":[72,89],"assignment,":[73],"table":[75],"lookup":[76],"select":[77],"local":[78],"or":[79],"remote":[80],"prefill.":[81],"evaluate":[83],"DolphinRouting":[84,107],"real":[87,98,105],"PD-disaggregated":[88],"using":[90],"three":[91],"public":[92],"benchmarks,":[93],"mixed":[95,102],"workload,":[96],"data.":[99],"Across":[100],"workload":[103],"data,":[106],"with":[108],"$k$-means":[109],"reduces":[110],"P90":[111,116],"TTFT":[112],"5.7--7.9\\%":[114],"TPOT":[117],"2.6--10.1\\%":[119],"relative":[120],"to":[121],"Dynamo.":[122]},"counts_by_year":[],"updated_date":"2026-11-02T09:45:03.842371","created_date":"2026-10-31T00:00:00"}