{"id":"https://openalex.org/W4416712272","doi":"https://doi.org/10.48550/arxiv.2509.02718","title":"Efficient Training-Free Online Routing for High-Volume Multi-LLM Serving","display_name":"Efficient Training-Free Online Routing for High-Volume Multi-LLM Serving","publication_year":2025,"publication_date":"2025-09-02","ids":{"openalex":"https://openalex.org/W4416712272","doi":"https://doi.org/10.48550/arxiv.2509.02718"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2509.02718","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.02718","pdf_url":"https://arxiv.org/pdf/2509.02718","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2509.02718","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074041862","display_name":"Fangzhou Wu","orcid":"https://orcid.org/0000-0002-1070-842X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Fangzhou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5120476675","display_name":"Sandeep Silwal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Silwal, Sandeep","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.22550000250339508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.22550000250339508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.1151999980211258,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.06019999831914902,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6462000012397766},{"id":"https://openalex.org/keywords/routing","display_name":"Routing (electronic design automation)","score":0.5346999764442444},{"id":"https://openalex.org/keywords/online-algorithm","display_name":"Online algorithm","score":0.5227000117301941},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4936999976634979},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.48429998755455017},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.4668000042438507},{"id":"https://openalex.org/keywords/static-routing","display_name":"Static routing","score":0.45879998803138733},{"id":"https://openalex.org/keywords/routing-table","display_name":"Routing table","score":0.4359000027179718},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.42239999771118164},{"id":"https://openalex.org/keywords/competitive-analysis","display_name":"Competitive analysis","score":0.3783000111579895}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8090999722480774},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6462000012397766},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.5346999764442444},{"id":"https://openalex.org/C196921405","wikidata":"https://www.wikidata.org/wiki/Q786431","display_name":"Online algorithm","level":2,"score":0.5227000117301941},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4936999976634979},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.48429998755455017},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.4668000042438507},{"id":"https://openalex.org/C204948658","wikidata":"https://www.wikidata.org/wiki/Q1119410","display_name":"Static routing","level":4,"score":0.45879998803138733},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.43939998745918274},{"id":"https://openalex.org/C184896649","wikidata":"https://www.wikidata.org/wiki/Q290066","display_name":"Routing table","level":4,"score":0.4359000027179718},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.42239999771118164},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.4041999876499176},{"id":"https://openalex.org/C102408133","wikidata":"https://www.wikidata.org/wiki/Q5156350","display_name":"Competitive analysis","level":3,"score":0.3783000111579895},{"id":"https://openalex.org/C20556612","wikidata":"https://www.wikidata.org/wiki/Q4469374","display_name":"Volume (thermodynamics)","level":2,"score":0.37689998745918274},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.3727000057697296},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.35370001196861267},{"id":"https://openalex.org/C9659607","wikidata":"https://www.wikidata.org/wiki/Q1268903","display_name":"Dynamic Source Routing","level":4,"score":0.3506999909877777},{"id":"https://openalex.org/C196423136","wikidata":"https://www.wikidata.org/wiki/Q7209671","display_name":"Policy-based routing","level":5,"score":0.35019999742507935},{"id":"https://openalex.org/C29436982","wikidata":"https://www.wikidata.org/wiki/Q3700557","display_name":"Destination-Sequenced Distance Vector routing","level":5,"score":0.3393999934196472},{"id":"https://openalex.org/C157692150","wikidata":"https://www.wikidata.org/wiki/Q2919848","display_name":"Query optimization","level":2,"score":0.3346000015735626},{"id":"https://openalex.org/C50558702","wikidata":"https://www.wikidata.org/wiki/Q5535067","display_name":"Geographic routing","level":5,"score":0.33230000734329224},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3271999955177307},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3197999894618988},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.289900004863739},{"id":"https://openalex.org/C89305328","wikidata":"https://www.wikidata.org/wiki/Q1755411","display_name":"Link-state routing protocol","level":4,"score":0.27889999747276306},{"id":"https://openalex.org/C125583679","wikidata":"https://www.wikidata.org/wiki/Q755673","display_name":"Search algorithm","level":2,"score":0.275299996137619},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.2752000093460083},{"id":"https://openalex.org/C104954878","wikidata":"https://www.wikidata.org/wiki/Q1648707","display_name":"Routing protocol","level":3,"score":0.2646999955177307},{"id":"https://openalex.org/C192028432","wikidata":"https://www.wikidata.org/wiki/Q845739","display_name":"Query language","level":2,"score":0.2623000144958496},{"id":"https://openalex.org/C99016210","wikidata":"https://www.wikidata.org/wiki/Q5488129","display_name":"Query expansion","level":2,"score":0.25360000133514404},{"id":"https://openalex.org/C118689300","wikidata":"https://www.wikidata.org/wiki/Q7978614","display_name":"Web query classification","level":4,"score":0.25290000438690186}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2509.02718","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.02718","pdf_url":"https://arxiv.org/pdf/2509.02718","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2509.02718","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2509.02718","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2509.02718","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.02718","pdf_url":"https://arxiv.org/pdf/2509.02718","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Increasing":[0],"demand":[1],"for":[2,67],"Large":[3],"Language":[4],"Models":[5],"(LLMs)":[6],"services":[7],"imposes":[8],"substantial":[9],"deployment":[10],"and":[11,32,43,54,83,134,150],"computation":[12],"costs":[13],"on":[14,30,40],"providers.":[15],"LLM":[16,28],"routing":[17,69,98],"offers":[18],"a":[19,85,89,97,113],"cost-efficient":[20],"solution":[21],"by":[22,127],"directing":[23],"queries":[24,94],"to":[25,45,47,78,95],"the":[26,63],"optimal":[27],"based":[29],"model":[31],"query":[33,52,81],"features.":[34],"However,":[35],"existing":[36],"works":[37],"primarily":[38],"focus":[39],"offline":[41],"scenarios":[42],"struggle":[44],"adapt":[46],"online":[48,68],"settings":[49],"with":[50],"high":[51],"volume":[53],"constrained":[55],"token":[56],"budgets.":[57],"In":[58],"this":[59],"work,":[60],"we":[61],"introduce":[62],"first":[64],"training-free":[65],"algorithm":[66,72,111],"scenarios.":[70],"Our":[71,155],"leverages":[73],"approximate":[74],"nearest":[75],"neighbor":[76],"search":[77],"efficiently":[79],"estimate":[80],"features":[82],"performs":[84],"one-time":[86],"optimization":[87],"over":[88],"small":[90],"set":[91],"of":[92,116,141],"initial":[93],"learn":[96],"strategy":[99],"that":[100,109],"guides":[101],"future":[102],"routing.":[103],"We":[104],"provide":[105],"theoretical":[106],"guarantees":[107],"demonstrating":[108],"our":[110],"achieves":[112],"competitive":[114],"ratio":[115],"$1":[117],"-":[118],"o(1)$":[119],"under":[120],"natural":[121],"assumptions,":[122],"which":[123],"is":[124,157],"further":[125],"validated":[126],"extensive":[128],"experiments":[129],"across":[130],"3":[131],"benchmark":[132],"datasets":[133],"8":[135],"baselines,":[136],"showing":[137],"an":[138],"average":[139],"improvement":[140],"3.55$\\times$":[142],"in":[143,147,153],"overall":[144],"performance,":[145],"1.85$\\times$":[146],"cost":[148],"efficiency,":[149],"nearly":[151],"4.25$\\times$":[152],"throughput.":[154],"code":[156],"available":[158],"at":[159],"https://github.com/fzwark/PORT.":[160]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-10-10T00:00:00"}