{"id":"https://openalex.org/W7154396947","doi":"https://doi.org/10.48550/arxiv.2604.09603","title":"ECHO: Elastic Speculative Decoding with Sparse Gating for High-Concurrency Scenarios","display_name":"ECHO: Elastic Speculative Decoding with Sparse Gating for High-Concurrency Scenarios","publication_year":2026,"publication_date":"2026-03-10","ids":{"openalex":"https://openalex.org/W7154396947","doi":"https://doi.org/10.48550/arxiv.2604.09603"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.09603","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.09603","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.09603","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133568879","display_name":"Xinyi Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Xinyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133581089","display_name":"Yuhao Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Yuhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133570218","display_name":"Baolin Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Baolin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133607979","display_name":"Hengxin Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Hengxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133564704","display_name":"Jun Dai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dai, Jun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133611428","display_name":"Shuang Ge","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ge, Shuang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133561291","display_name":"Lei Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Lei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133584861","display_name":"Yue Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5108223442","display_name":"Mingcheng Wan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wan, Mingcheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.3871000111103058,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.3871000111103058,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10904","display_name":"Embedded Systems Design Techniques","score":0.23080000281333923,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.10459999740123749,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.8364999890327454},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.6215999722480774},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5292999744415283},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.5134999752044678},{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.4117000102996826},{"id":"https://openalex.org/keywords/gating","display_name":"Gating","score":0.34450000524520874},{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.33809998631477356},{"id":"https://openalex.org/keywords/computational-complexity-theory","display_name":"Computational complexity theory","score":0.3312000036239624}],"concepts":[{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.8364999890327454},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7698000073432922},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.6215999722480774},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5407999753952026},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5292999744415283},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.5134999752044678},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.4117000102996826},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3804999887943268},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3725999891757965},{"id":"https://openalex.org/C194544171","wikidata":"https://www.wikidata.org/wiki/Q21105679","display_name":"Gating","level":2,"score":0.34450000524520874},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.33809998631477356},{"id":"https://openalex.org/C179799912","wikidata":"https://www.wikidata.org/wiki/Q205084","display_name":"Computational complexity theory","level":2,"score":0.3312000036239624},{"id":"https://openalex.org/C56372850","wikidata":"https://www.wikidata.org/wiki/Q1050404","display_name":"Sparse matrix","level":3,"score":0.3253999948501587},{"id":"https://openalex.org/C2777472644","wikidata":"https://www.wikidata.org/wiki/Q16968992","display_name":"Approximate inference","level":3,"score":0.2976999878883362},{"id":"https://openalex.org/C2984822820","wikidata":"https://www.wikidata.org/wiki/Q1123036","display_name":"Processor scheduling","level":3,"score":0.2768000066280365},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.27469998598098755},{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.27079999446868896},{"id":"https://openalex.org/C2779426996","wikidata":"https://www.wikidata.org/wiki/Q18389128","display_name":"Echo (communications protocol)","level":2,"score":0.260699987411499},{"id":"https://openalex.org/C162838799","wikidata":"https://www.wikidata.org/wiki/Q596077","display_name":"Counterexample","level":2,"score":0.25929999351501465},{"id":"https://openalex.org/C51823790","wikidata":"https://www.wikidata.org/wiki/Q504353","display_name":"Greedy algorithm","level":2,"score":0.25270000100135803},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.2524000108242035},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.2506999969482422}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.09603","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.09603","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.09603","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.09603","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Speculative":[0],"Decoding":[1],"promises":[2],"to":[3,87,102,138],"accelerate":[4],"the":[5,23,33,89,104,121],"inference":[6],"of":[7,26],"Large":[8],"Language":[9],"Models,":[10],"yet":[11],"its":[12],"efficacy":[13],"often":[14],"degrades":[15],"in":[16,130],"production-grade":[17],"serving.":[18],"Existing":[19],"evaluations":[20,116],"typically":[21],"overlook":[22],"compute-bound":[24],"nature":[25],"high-concurrency":[27],"regimes,":[28],"where":[29],"verification":[30,46,109],"compute":[31],"becomes":[32],"dominant":[34],"bottleneck.":[35],"Consequently,":[36],"prior":[37],"methods":[38,129],"face":[39],"a":[40,65,77,92],"dilemma:":[41],"static":[42],"trees":[43,50],"incur":[44],"massive":[45],"waste,":[47],"while":[48],"dynamic":[49],"suffer":[51],"from":[52],"cumulative":[53],"misjudgments":[54],"and":[55,100,111,133,142],"kernel":[56],"incompatibility.":[57],"To":[58],"bridge":[59],"this":[60],"gap,":[61],"we":[62],"introduce":[63],"ECHO,":[64],"high":[66],"concurrency-oriented":[67],"framework":[68],"integrated":[69],"into":[70],"SGLang":[71],"that":[72,124],"reformulates":[73],"speculative":[74],"execution":[75],"as":[76,91],"budgeted":[78],"scheduling":[79],"problem.":[80],"Crucially,":[81],"ECHO":[82,125],"employs":[83],"sparse":[84],"confidence":[85],"gating":[86],"manage":[88],"batch":[90],"unified":[93],"super-tree,":[94],"elastically":[95],"pivoting":[96],"budget":[97],"between":[98,106],"depth":[99],"width":[101],"co-optimize":[103],"trade-off":[105],"reducing":[107],"global":[108],"steps":[110],"maximizing":[112],"per-step":[113],"efficiency.":[114],"Extensive":[115],"across":[117],"diverse":[118],"model":[119],"scales-particularly":[120],"industrial-grade":[122],"Qwen3-235B-demonstrate":[123],"consistently":[126],"outperforms":[127],"SOTA":[128],"both":[131],"low-load":[132],"high-load":[134],"scenarios,":[135],"achieving":[136],"up":[137],"5.35x":[139],"walltime":[140],"speedup":[141,147],"delivering":[143],"over":[144],"20%":[145],"relative":[146],"gain.":[148]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-15T00:00:00"}