{"id":"https://openalex.org/W7136170912","doi":"https://doi.org/10.48550/arxiv.2603.12646","title":"98$\\times$ Faster LLM Routing Without a Dedicated GPU: Flash Attention, Prompt Compression, and Near-Streaming for the vLLM Semantic Router","display_name":"98$\\times$ Faster LLM Routing Without a Dedicated GPU: Flash Attention, Prompt Compression, and Near-Streaming for the vLLM Semantic Router","publication_year":2026,"publication_date":"2026-03-13","ids":{"openalex":"https://openalex.org/W7136170912","doi":"https://doi.org/10.48550/arxiv.2603.12646"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.12646","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.12646","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.12646","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030019120","display_name":"X Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xunzhuo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129586241","display_name":"Bowei He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Bowei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129606253","display_name":"Xue Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129563769","display_name":"Andy Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Andy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129552855","display_name":"Haichen Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Haichen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129558744","display_name":"Huamin Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Huamin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12326","display_name":"Network Packet Processing and Optimization","score":0.8323000073432922,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12326","display_name":"Network Packet Processing and Optimization","score":0.8323000073432922,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10714","display_name":"Software-Defined Networks and 5G","score":0.09380000084638596,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12292","display_name":"Graph Theory and Algorithms","score":0.006800000090152025,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.6425999999046326},{"id":"https://openalex.org/keywords/router","display_name":"Router","score":0.5701000094413757},{"id":"https://openalex.org/keywords/memory-footprint","display_name":"Memory footprint","score":0.5497000217437744},{"id":"https://openalex.org/keywords/chunking","display_name":"Chunking (psychology)","score":0.38119998574256897},{"id":"https://openalex.org/keywords/cas-latency","display_name":"CAS latency","score":0.3806000053882599},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.373199999332428},{"id":"https://openalex.org/keywords/network-packet","display_name":"Network packet","score":0.3531999886035919},{"id":"https://openalex.org/keywords/serialization","display_name":"Serialization","score":0.3147999942302704},{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.3140999972820282},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.3140000104904175}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8834999799728394},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.6425999999046326},{"id":"https://openalex.org/C2775896111","wikidata":"https://www.wikidata.org/wiki/Q642560","display_name":"Router","level":2,"score":0.5701000094413757},{"id":"https://openalex.org/C74912251","wikidata":"https://www.wikidata.org/wiki/Q6815727","display_name":"Memory footprint","level":2,"score":0.5497000217437744},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3944999873638153},{"id":"https://openalex.org/C203357204","wikidata":"https://www.wikidata.org/wiki/Q1089605","display_name":"Chunking (psychology)","level":2,"score":0.38119998574256897},{"id":"https://openalex.org/C189930140","wikidata":"https://www.wikidata.org/wiki/Q1112878","display_name":"CAS latency","level":4,"score":0.3806000053882599},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.37950000166893005},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.373199999332428},{"id":"https://openalex.org/C158379750","wikidata":"https://www.wikidata.org/wiki/Q214111","display_name":"Network packet","level":2,"score":0.3531999886035919},{"id":"https://openalex.org/C52723943","wikidata":"https://www.wikidata.org/wiki/Q1127410","display_name":"Serialization","level":2,"score":0.3147999942302704},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.3140999972820282},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.3140000104904175},{"id":"https://openalex.org/C2778000800","wikidata":"https://www.wikidata.org/wiki/Q830043","display_name":"Handshake","level":3,"score":0.30820000171661377},{"id":"https://openalex.org/C2778119891","wikidata":"https://www.wikidata.org/wiki/Q477690","display_name":"CUDA","level":2,"score":0.3059999942779541},{"id":"https://openalex.org/C25344961","wikidata":"https://www.wikidata.org/wiki/Q192726","display_name":"Virtual machine","level":2,"score":0.30250000953674316},{"id":"https://openalex.org/C2780416260","wikidata":"https://www.wikidata.org/wiki/Q2063","display_name":"JSON","level":2,"score":0.3021000027656555},{"id":"https://openalex.org/C2781357197","wikidata":"https://www.wikidata.org/wiki/Q5757597","display_name":"High memory","level":2,"score":0.30000001192092896},{"id":"https://openalex.org/C188045654","wikidata":"https://www.wikidata.org/wiki/Q17148339","display_name":"Memory bandwidth","level":2,"score":0.29589998722076416},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.295199990272522},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.29499998688697815},{"id":"https://openalex.org/C159379195","wikidata":"https://www.wikidata.org/wiki/Q7239568","display_name":"Precomputation","level":3,"score":0.2935999929904938},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.29350000619888306},{"id":"https://openalex.org/C82687282","wikidata":"https://www.wikidata.org/wiki/Q66221","display_name":"Auxiliary memory","level":2,"score":0.29249998927116394},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.28940001130104065},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.2888999879360199},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2888000011444092},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.28439998626708984},{"id":"https://openalex.org/C28362024","wikidata":"https://www.wikidata.org/wiki/Q2067413","display_name":"Thrashing","level":2,"score":0.2827000021934509},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.273499995470047},{"id":"https://openalex.org/C176649486","wikidata":"https://www.wikidata.org/wiki/Q2308807","display_name":"Memory management","level":3,"score":0.2702000141143799},{"id":"https://openalex.org/C48105269","wikidata":"https://www.wikidata.org/wiki/Q1141160","display_name":"Header","level":2,"score":0.2635999917984009},{"id":"https://openalex.org/C184896649","wikidata":"https://www.wikidata.org/wiki/Q290066","display_name":"Routing table","level":4,"score":0.258899986743927},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.25519999861717224},{"id":"https://openalex.org/C9390403","wikidata":"https://www.wikidata.org/wiki/Q3966","display_name":"Computer hardware","level":1,"score":0.2547000050544739},{"id":"https://openalex.org/C162262903","wikidata":"https://www.wikidata.org/wiki/Q343527","display_name":"Allocator","level":2,"score":0.251800000667572},{"id":"https://openalex.org/C76399640","wikidata":"https://www.wikidata.org/wiki/Q189401","display_name":"Virtual memory","level":4,"score":0.250900000333786}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.12646","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.12646","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.12646","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.12646","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"System-level":[0],"routers":[1],"that":[2,101],"intercept":[3],"LLM":[4,42,224],"requests":[5],"for":[6,41,75,91,117,230],"safety":[7],"classification,":[8],"domain":[9],"routing,":[10],"and":[11,18,106,129,153,168,189,208,226],"PII":[12],"detection":[13],"must":[14],"be":[15],"both":[16,103,166],"fast":[17],"operationally":[19],"lightweight:":[20],"they":[21],"should":[22],"add":[23],"minimal":[24],"latency":[25,105,132,167],"to":[26,127,135,159,202,219],"every":[27],"request,":[28],"yet":[29],"not":[30],"require":[31],"a":[32,111,172,209,221,231],"dedicated":[33,232],"GPU":[34,52,169,212,222],"--":[35,216],"an":[36],"expensive":[37],"resource":[38],"better":[39],"used":[40],"inference":[43],"itself.":[44],"When":[45],"the":[46,50,81,92,104,107,228],"router":[47,211],"co-locates":[48],"on":[49,97,120],"same":[51],"as":[53],"vLLM":[54,93],"serving":[55,225],"instances,":[56],"standard":[57],"attention's":[58],"$O(n^2)$":[59,126],"memory":[60,82,108,124,170],"makes":[61],"long-context":[62],"classification":[63],"(8K--32K":[64],"tokens)":[65],"impossible:":[66],"at":[67,171],"8K":[68],"tokens,":[69],"three":[70,88],"concurrent":[71],"classifiers":[72],"need":[73,229],"${\\sim}$4.5\\,GB":[74],"attention":[76,123],"masks":[77],"alone,":[78],"far":[79],"exceeding":[80],"left":[83],"by":[84],"vLLM.":[85],"We":[86],"present":[87],"staged":[89],"optimizations":[90],"Semantic":[94],"Router,":[95],"benchmarked":[96],"AMD":[98,236],"Instinct":[99],"MI300X,":[100],"solve":[102],"problem.":[109],"\\emph{Stage~1}:":[110],"custom":[112],"CK":[113],"Flash":[114],"Attention":[115],"operator":[116],"ONNX":[118],"Runtime":[119],"ROCm":[121,237],"reduces":[122,156],"from":[125,133],"$O(n)$":[128],"end-to-end":[130],"(E2E)":[131],"4{,}918\\,ms":[134],"127\\,ms":[136],"(\\textbf{38.7$\\times$}),":[137],"enabling":[138],"8K--32K":[139],"tokens":[140,161],"where":[141],"SDPA":[142],"OOMs.":[143],"\\emph{Stage~2}:":[144],"classical":[145],"NLP":[146],"prompt":[147,177],"compression":[148],"(TextRank,":[149],"position":[150],"weighting,":[151],"TF-IDF,":[152],"novelty":[154],"scoring)":[155],"all":[157],"inputs":[158],"${\\sim}$512":[160],"without":[162],"neural":[163],"inference,":[164],"capping":[165],"constant":[173],"regardless":[174],"of":[175],"original":[176],"length":[178],"(E2E":[179,195],"127$\\to$62\\,ms,":[180],"\\textbf{2.0$\\times$}).":[181],"\\emph{Stage~3}:":[182],"near-streaming":[183],"body":[184],"processing":[185],"with":[186,223],"adaptive":[187],"chunking":[188],"zero-copy":[190],"JSON":[191],"eliminates":[192],"serialization":[193],"overhead":[194],"62$\\to$50\\,ms,":[196],"\\textbf{1.2$\\times$}).":[197],"Cumulatively:":[198],"\\textbf{98$\\times$}":[199],"improvement":[200],"(4{,}918\\,ms":[201],"50\\,ms),":[203],"16K-token":[204],"routing":[205],"in":[206],"108\\,ms,":[207],"total":[210],"footprint":[213],"under":[214],"800\\,MB":[215],"small":[217],"enough":[218],"share":[220],"removing":[227],"accelerator.":[233],"Stage~1":[234],"targets":[235],"(NVIDIA":[238],"GPUs":[239],"already":[240],"have":[241],"FlashAttention":[242],"via":[243],"cuDNN);":[244],"Stages~2":[245],"and~3":[246],"are":[247],"hardware-agnostic.":[248]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-03-17T00:00:00"}