{"id":"https://openalex.org/W7134898416","doi":"https://doi.org/10.1145/3779212.3790246","title":"SwiftSpec: Disaggregated Speculative Decoding and Fused Kernels for Low-Latency LLM Inference","display_name":"SwiftSpec: Disaggregated Speculative Decoding and Fused Kernels for Low-Latency LLM Inference","publication_year":2026,"publication_date":"2026-03-10","ids":{"openalex":"https://openalex.org/W7134898416","doi":"https://doi.org/10.1145/3779212.3790246"},"language":null,"primary_location":{"id":"doi:10.1145/3779212.3790246","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3779212.3790246","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 2","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3779212.3790246","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Ziyi Zhang","orcid":"https://orcid.org/0009-0004-4336-6341"},"institutions":[{"id":"https://openalex.org/I39422238","display_name":"University of Illinois Chicago","ror":"https://ror.org/02mpq6x41","country_code":"US","type":"education","lineage":["https://openalex.org/I39422238"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ziyi Zhang","raw_affiliation_strings":["ByteDance Seed, Bellevue, WA, USA and University of Chicago, Chicago, IL, USA"],"raw_orcid":"https://orcid.org/0009-0004-4336-6341","affiliations":[{"raw_affiliation_string":"ByteDance Seed, Bellevue, WA, USA and University of Chicago, Chicago, IL, USA","institution_ids":["https://openalex.org/I39422238"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128776818","display_name":"Ziheng Jiang","orcid":null},"institutions":[{"id":"https://openalex.org/I174135032","display_name":"Bellevue College","ror":"https://ror.org/05gr4yv49","country_code":"US","type":"education","lineage":["https://openalex.org/I174135032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ziheng Jiang","raw_affiliation_strings":["ByteDance Seed, Bellevue, WA, USA"],"raw_orcid":"https://orcid.org/0009-0001-7732-4391","affiliations":[{"raw_affiliation_string":"ByteDance Seed, Bellevue, WA, USA","institution_ids":["https://openalex.org/I174135032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021566476","display_name":"Chengquan Jiang","orcid":"https://orcid.org/0009-0004-9356-6034"},"institutions":[{"id":"https://openalex.org/I174135032","display_name":"Bellevue College","ror":"https://ror.org/05gr4yv49","country_code":"US","type":"education","lineage":["https://openalex.org/I174135032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chengquan Jiang","raw_affiliation_strings":["ByteDance Seed, Bellevue, WA, USA"],"raw_orcid":"https://orcid.org/0009-0004-9356-6034","affiliations":[{"raw_affiliation_string":"ByteDance Seed, Bellevue, WA, USA","institution_ids":["https://openalex.org/I174135032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128785745","display_name":"Menghan Yu","orcid":null},"institutions":[{"id":"https://openalex.org/I174135032","display_name":"Bellevue College","ror":"https://ror.org/05gr4yv49","country_code":"US","type":"education","lineage":["https://openalex.org/I174135032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Menghan Yu","raw_affiliation_strings":["ByteDance Seed, Bellevue, WA, USA"],"raw_orcid":"https://orcid.org/0009-0008-8240-8144","affiliations":[{"raw_affiliation_string":"ByteDance Seed, Bellevue, WA, USA","institution_ids":["https://openalex.org/I174135032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128704697","display_name":"Size Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Size Zheng","raw_affiliation_strings":["ByteDance Seed, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9471-1780","affiliations":[{"raw_affiliation_string":"ByteDance Seed, Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128734372","display_name":"Haibin Lin","orcid":null},"institutions":[{"id":"https://openalex.org/I174135032","display_name":"Bellevue College","ror":"https://ror.org/05gr4yv49","country_code":"US","type":"education","lineage":["https://openalex.org/I174135032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Haibin Lin","raw_affiliation_strings":["ByteDance Seed, Bellevue, WA, USA"],"raw_orcid":"https://orcid.org/0000-0003-4879-5335","affiliations":[{"raw_affiliation_string":"ByteDance Seed, Bellevue, WA, USA","institution_ids":["https://openalex.org/I174135032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128798711","display_name":"Xin Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I174135032","display_name":"Bellevue College","ror":"https://ror.org/05gr4yv49","country_code":"US","type":"education","lineage":["https://openalex.org/I174135032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xin Liu","raw_affiliation_strings":["ByteDance Seed, Bellevue, WA, USA"],"raw_orcid":"https://orcid.org/0009-0000-8346-3323","affiliations":[{"raw_affiliation_string":"ByteDance Seed, Bellevue, WA, USA","institution_ids":["https://openalex.org/I174135032"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5126788517","display_name":"Henry Hoffmann","orcid":null},"institutions":[{"id":"https://openalex.org/I39422238","display_name":"University of Illinois Chicago","ror":"https://ror.org/02mpq6x41","country_code":"US","type":"education","lineage":["https://openalex.org/I39422238"]},{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Henry Hoffmann","raw_affiliation_strings":["University of Chicago, Chicago, IL, USA"],"raw_orcid":"https://orcid.org/0000-0003-0816-8150","affiliations":[{"raw_affiliation_string":"University of Chicago, Chicago, IL, USA","institution_ids":["https://openalex.org/I39422238","https://openalex.org/I40347166"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2197","last_page":"2211"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.24969999492168427,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.24969999492168427,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.1429000049829483,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.13950000703334808,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.6625000238418579},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.6129999756813049},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.48919999599456787},{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.46070000529289246},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.41190001368522644},{"id":"https://openalex.org/keywords/reuse","display_name":"Reuse","score":0.4047999978065491},{"id":"https://openalex.org/keywords/instruction-set","display_name":"Instruction set","score":0.38850000500679016},{"id":"https://openalex.org/keywords/multi-core-processor","display_name":"Multi-core processor","score":0.3659000098705292},{"id":"https://openalex.org/keywords/sequential-decoding","display_name":"Sequential decoding","score":0.3596999943256378}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8140000104904175},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.7583000063896179},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.6625000238418579},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.6129999756813049},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.48919999599456787},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.46070000529289246},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.41190001368522644},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.4047999978065491},{"id":"https://openalex.org/C202491316","wikidata":"https://www.wikidata.org/wiki/Q272683","display_name":"Instruction set","level":2,"score":0.38850000500679016},{"id":"https://openalex.org/C78766204","wikidata":"https://www.wikidata.org/wiki/Q555032","display_name":"Multi-core processor","level":2,"score":0.3659000098705292},{"id":"https://openalex.org/C193969084","wikidata":"https://www.wikidata.org/wiki/Q7452500","display_name":"Sequential decoding","level":4,"score":0.3596999943256378},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.3582000136375427},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3580000102519989},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3546000123023987},{"id":"https://openalex.org/C62611344","wikidata":"https://www.wikidata.org/wiki/Q1062658","display_name":"Node (physics)","level":2,"score":0.34040001034736633},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.3377000093460083},{"id":"https://openalex.org/C107598950","wikidata":"https://www.wikidata.org/wiki/Q259864","display_name":"Microarchitecture","level":2,"score":0.33079999685287476},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.33009999990463257},{"id":"https://openalex.org/C2778119891","wikidata":"https://www.wikidata.org/wiki/Q477690","display_name":"CUDA","level":2,"score":0.32690000534057617},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.3246000111103058},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.3034999966621399},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2913999855518341},{"id":"https://openalex.org/C46637626","wikidata":"https://www.wikidata.org/wiki/Q6693015","display_name":"Low latency (capital markets)","level":2,"score":0.28029999136924744},{"id":"https://openalex.org/C140745168","wikidata":"https://www.wikidata.org/wiki/Q1210082","display_name":"Tree traversal","level":2,"score":0.2703000009059906},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.2687000036239624},{"id":"https://openalex.org/C204397858","wikidata":"https://www.wikidata.org/wiki/Q4437907","display_name":"List decoding","level":5,"score":0.26809999346733093},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.2669000029563904},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.25459998846054077}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3779212.3790246","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3779212.3790246","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 2","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3779212.3790246","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3779212.3790246","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 2","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":7,"referenced_works":["https://openalex.org/W2912924812","https://openalex.org/W4395112660","https://openalex.org/W4401176373","https://openalex.org/W4401414182","https://openalex.org/W4405756215","https://openalex.org/W4413145951","https://openalex.org/W7133227460"],"related_works":[],"abstract_inverted_index":{"Low-latency,":[0],"single-request":[1,57],"decoding":[2,22,150,154],"of":[3,92,103],"large":[4],"language":[5],"models":[6,75],"is":[7,169],"critical":[8,40],"for":[9,106,114,135,166,185,190],"interactive":[10],"systems":[11],"with":[12,28,56],"tight":[13],"SLA":[14],"demands.":[15],"Prior":[16],"work":[17],"reduces":[18],"latency":[19],"through":[20],"speculative":[21,153],"(combining":[23],"a":[24,29,67,80,126,143],"small":[25,52],"draft":[26,35,72,121],"model":[27,36,157],"larger":[30],"target":[31,74,123],"model),":[32],"but":[33],"the":[34,39,51,90,181],"remains":[37],"on":[38,142],"path,":[41],"and":[42,73,83,95,109,117,122,159],"communication":[43],"overhead":[44],"limits":[45],"scaling":[46],"across":[47,76,155,172],"GPUs":[48,78],"due":[49],"to":[50,88],"batch":[53],"size":[54],"associated":[55],"decoding.":[58],"To":[59],"address":[60],"these":[61],"limitations,":[62],"this":[63],"paper":[64],"introduces":[65],"SwiftSpec:":[66],"system":[68],"architecture":[69],"that":[70,165],"disaggregates":[71],"homogeneous":[77],"within":[79],"single":[81,127],"node":[82],"utilizes":[84],"NCCL-low-latency":[85],"primitives":[86],"directly":[87],"improve":[89],"performance":[91],"core":[93],"GEMM":[94],"attention":[96],"kernels.":[97],"Our":[98],"implementation":[99],"includes":[100],"3k":[101],"lines":[102],"custom":[104],"CUDA":[105],"fused":[107],"kernels":[108],"an":[110],"evolving":[111],"tree":[112],"cache":[113],"KV-cache":[115],"consistency":[116],"maximized":[118],"reuse":[119],"between":[120],"models.":[124],"On":[125],"8\u00d7H800":[128],"GPU":[129],"node,":[130],"SwiftSpec":[131,168,191],"achieves":[132],"347":[133],"tokens/s":[134],"Llama-3-70B---1.3\u00d7":[136],"faster":[137,149,171],"than":[138,151],"NVIDIA's":[139],"own":[140],"benchmarks":[141],"higher-performance":[144],"8\u00d7H200":[145],"setup---and":[146],"averages":[147],"1.75\u00d7":[148],"state-of-the-art":[152],"five":[156],"families":[158],"six":[160],"datasets.":[161],"Specifically,":[162],"we":[163],"find":[164],"Llama-3-70B":[167],"significantly":[170],"all":[173],"480":[174],"tested":[175],"queries,":[176],"showing":[177],"1.7\u00d7":[178],"speedup":[179],"over":[180],"best":[182],"open-source":[183],"baseline":[184],"95th":[186],"percentile":[187],"requests.":[188],"Code":[189],"will":[192],"be":[193],"available":[194],"at":[195],"https://github.com/ByteDance-Seed/SwiftSpec":[196]},"counts_by_year":[],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2026-03-12T00:00:00"}