{"id":"https://openalex.org/W7166589956","doi":"https://doi.org/10.48550/arxiv.2606.27457","title":"Cluster, Route, Escalate: Cascaded Framework for Cost-Aware LLM Serving","display_name":"Cluster, Route, Escalate: Cascaded Framework for Cost-Aware LLM Serving","publication_year":2026,"publication_date":"2026-06-25","ids":{"openalex":"https://openalex.org/W7166589956","doi":"https://doi.org/10.48550/arxiv.2606.27457"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.27457","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.27457","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.27457","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003040447","display_name":"Yasmin Moslem","orcid":"https://orcid.org/0000-0003-4595-6877"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Moslem, Yasmin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046545440","display_name":"Magdalena Kacmajor","orcid":"https://orcid.org/0000-0001-6843-9050"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kacmajor, Magdalena","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021204772","display_name":"Vasudevan Nedumpozhimana","orcid":"https://orcid.org/0000-0001-5161-8925"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nedumpozhimana, Vasudevan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139592739","display_name":"Ammar Abbas","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Abbas, Ammar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107852633","display_name":"Shirin Panahi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Panahi, Solmaz","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090649387","display_name":"David Lynch","orcid":"https://orcid.org/0000-0002-0141-9781"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lynch, David","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139597405","display_name":"Zhuangzhuang Nie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nie, Zhuangzhuang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069334140","display_name":"Alexandros Agapitos","orcid":"https://orcid.org/0000-0001-7529-681X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Agapitos, Alexandros","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059314409","display_name":"Aleksandar Milenovi\u0107","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Milenovic, Aleksandar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139625283","display_name":"Hongmeng Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Hongmeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100902783","display_name":"Y Shi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shi, Yucheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139567286","display_name":"Yue Pan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pan, Yue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014264993","display_name":"Patricia Buffini","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Buffini, Patricia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139586819","display_name":"John D. Kelleher","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kelleher, John D.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.6603000164031982,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.6603000164031982,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.07320000231266022,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.02500000037252903,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6313999891281128},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.6079000234603882},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5975000262260437},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.49050000309944153},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4805999994277954},{"id":"https://openalex.org/keywords/production","display_name":"Production (economics)","score":0.460099995136261},{"id":"https://openalex.org/keywords/routing","display_name":"Routing (electronic design automation)","score":0.4169999957084656}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.732699990272522},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6313999891281128},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6079000234603882},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5975000262260437},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.49050000309944153},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4805999994277954},{"id":"https://openalex.org/C2778348673","wikidata":"https://www.wikidata.org/wiki/Q739302","display_name":"Production (economics)","level":2,"score":0.460099995136261},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.4169999957084656},{"id":"https://openalex.org/C164866538","wikidata":"https://www.wikidata.org/wiki/Q367351","display_name":"Cluster (spacecraft)","level":2,"score":0.37790000438690186},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.3723999857902527},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.34369999170303345},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.31839999556541443},{"id":"https://openalex.org/C146357865","wikidata":"https://www.wikidata.org/wiki/Q1123245","display_name":"Stage (stratigraphy)","level":2,"score":0.30869999527931213},{"id":"https://openalex.org/C8505890","wikidata":"https://www.wikidata.org/wiki/Q605095","display_name":"Budget constraint","level":2,"score":0.29249998927116394},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.29019999504089355},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.2879999876022339},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.28290000557899475},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.2646999955177307}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.27457","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.27457","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.27457","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.27457","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Efficient":[0],"deployment":[1],"of":[2,119],"large":[3],"language":[4],"models":[5],"(LLMs)":[6],"in":[7,140],"production":[8],"forces":[9],"a":[10,20,41,77,96],"trade-off":[11],"between":[12],"accuracy":[13,123],"and":[14,50,136],"cost.":[15],"Operators":[16],"often":[17],"default":[18],"to":[19,54,95,138],"single":[21],"model":[22,142],"that":[23],"is":[24,66,88,93],"either":[25],"expensive":[26,108],"for":[27,32,62],"easy":[28],"queries":[29,49],"or":[30,103],"insufficient":[31],"hard":[33,102],"ones.":[34],"To":[35],"address":[36],"this":[37,63],"challenge,":[38],"we":[39],"propose":[40],"two-stage":[42],"cascaded":[43,115],"solution.":[44],"Stage":[45,74,86],"1":[46,87],"clusters":[47],"incoming":[48],"assigns":[51],"each":[52],"cluster":[53],"its":[55],"most":[56],"cost-effective":[57],"model.":[58,98],"The":[59],"cost":[60],"budget":[61],"routing":[64],"process":[65],"set":[67],"by":[68],"an":[69,83],"interpretable":[70],"hyperparameter,":[71],"tuned":[72],"offline.":[73],"2":[75],"adds":[76],"quality":[78],"estimation":[79],"(QE)":[80],"cascade;":[81],"when":[82],"output":[84],"from":[85],"judged":[89],"low-quality,":[90],"the":[91,107,111,114,120,141],"query":[92],"escalated":[94],"stronger":[97],"This":[99],"ensures":[100],"only":[101,133],"low-confidence":[104],"cases":[105],"reach":[106],"models.":[109],"On":[110],"test":[112],"datasets,":[113],"system":[116],"retains":[117],"97-99%":[118],"strongest":[121],"model's":[122],"while":[124],"reducing":[125],"Time":[126],"Per":[127],"Output":[128],"Token":[129],"(TPOT).":[130],"It":[131],"requires":[132],"task-correctness":[134],"labels":[135],"adapts":[137],"changes":[139],"pool":[143],"without":[144],"manual":[145],"reconfiguration.":[146]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-30T00:00:00"}