{"id":"https://openalex.org/W7220458334","doi":"https://doi.org/10.48550/arxiv.2610.02800","title":"BitNest: Bit-Nested Speculative Decoding for Memory-Efficient LLM Inference Acceleration","display_name":"BitNest: Bit-Nested Speculative Decoding for Memory-Efficient LLM Inference Acceleration","publication_year":2026,"publication_date":"2026-10-02","ids":{"openalex":"https://openalex.org/W7220458334","doi":"https://doi.org/10.48550/arxiv.2610.02800"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.02800","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.02800","pdf_url":"https://arxiv.org/pdf/2610.02800","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.02800","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5121011462","display_name":"Chence Yang","orcid":"https://orcid.org/0009-0002-3678-6319","observed_orcids":["https://orcid.org/0009-0002-3678-6319"]},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]},{"id":"https://openalex.org/I165733156","display_name":"University of Georgia","ror":"https://ror.org/00te3t702","country_code":"US","type":"education","lineage":["https://openalex.org/I165733156"]},{"id":"https://openalex.org/I4401726789","display_name":"Xiaopeng Motors (China)","ror":"https://ror.org/05a5cgn70","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401726789"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Yang, Chence","raw_affiliation_strings":["Northeastern University","University of Georgia","XPeng Motors"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northeastern University","institution_ids":["https://openalex.org/I12912129"]},{"raw_affiliation_string":"University of Georgia","institution_ids":["https://openalex.org/I165733156"]},{"raw_affiliation_string":"XPeng Motors","institution_ids":["https://openalex.org/I4401726789"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134105177","display_name":"Ningxi Cheng","orcid":"https://orcid.org/0009-0007-2488-3150","observed_orcids":["https://orcid.org/0009-0007-2488-3150"]},"institutions":[{"id":"https://openalex.org/I165733156","display_name":"University of Georgia","ror":"https://ror.org/00te3t702","country_code":"US","type":"education","lineage":["https://openalex.org/I165733156"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cheng, Ningxi","raw_affiliation_strings":["University of Georgia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Georgia","institution_ids":["https://openalex.org/I165733156"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135994307","display_name":"Arash Akbari","orcid":"https://orcid.org/0009-0006-1846-0672","observed_orcids":["https://orcid.org/0009-0006-1846-0672"]},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Akbari, Arash","raw_affiliation_strings":["Northeastern University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northeastern University","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056636366","display_name":"Qitao Tan","orcid":"https://orcid.org/0000-0002-1280-0334","observed_orcids":["https://orcid.org/0000-0002-1280-0334"]},"institutions":[{"id":"https://openalex.org/I165733156","display_name":"University of Georgia","ror":"https://ror.org/00te3t702","country_code":"US","type":"education","lineage":["https://openalex.org/I165733156"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tan, Qitao","raw_affiliation_strings":["University of Georgia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Georgia","institution_ids":["https://openalex.org/I165733156"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125085192","display_name":"Qingchan Zhu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I165733156","display_name":"University of Georgia","ror":"https://ror.org/00te3t702","country_code":"US","type":"education","lineage":["https://openalex.org/I165733156"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhu, Qingchan","raw_affiliation_strings":["University of Georgia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Georgia","institution_ids":["https://openalex.org/I165733156"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5157376778","display_name":"Ci Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I165733156","display_name":"University of Georgia","ror":"https://ror.org/00te3t702","country_code":"US","type":"education","lineage":["https://openalex.org/I165733156"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhang, Ci","raw_affiliation_strings":["University of Georgia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Georgia","institution_ids":["https://openalex.org/I165733156"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033198334","display_name":"Changdi Yang","orcid":"https://orcid.org/0000-0002-8848-3806","observed_orcids":["https://orcid.org/0000-0002-8848-3806"]},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]},{"id":"https://openalex.org/I165733156","display_name":"University of Georgia","ror":"https://ror.org/00te3t702","country_code":"US","type":"education","lineage":["https://openalex.org/I165733156"]},{"id":"https://openalex.org/I4401726789","display_name":"Xiaopeng Motors (China)","ror":"https://ror.org/05a5cgn70","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401726789"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Yang, Changdi","raw_affiliation_strings":["Northeastern University","University of Georgia","XPeng Motors"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northeastern University","institution_ids":["https://openalex.org/I12912129"]},{"raw_affiliation_string":"University of Georgia","institution_ids":["https://openalex.org/I165733156"]},{"raw_affiliation_string":"XPeng Motors","institution_ids":["https://openalex.org/I4401726789"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100651384","display_name":"Yanzhi Wang","orcid":"https://orcid.org/0000-0002-3024-7990","observed_orcids":["https://orcid.org/0000-0002-3024-7990","https://orcid.org/0009-0009-3924-6466","https://orcid.org/0009-0008-4749-5860"]},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Yanzhi","raw_affiliation_strings":["Northeastern University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northeastern University","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043054935","display_name":"Wei Niu","orcid":"https://orcid.org/0000-0002-2697-7042","observed_orcids":["https://orcid.org/0000-0002-2697-7042"]},"institutions":[{"id":"https://openalex.org/I165733156","display_name":"University of Georgia","ror":"https://ror.org/00te3t702","country_code":"US","type":"education","lineage":["https://openalex.org/I165733156"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Niu, Wei","raw_affiliation_strings":["University of Georgia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Georgia","institution_ids":["https://openalex.org/I165733156"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5157536297","display_name":"Jinhui Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I17301866","display_name":"University of Alabama","ror":"https://ror.org/03xrrjk67","country_code":"US","type":"education","lineage":["https://openalex.org/I17301866"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Jinhui","raw_affiliation_strings":["The University of Alabama"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Alabama","institution_ids":["https://openalex.org/I17301866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5157485059","display_name":"Jin Lu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I165733156","display_name":"University of Georgia","ror":"https://ror.org/00te3t702","country_code":"US","type":"education","lineage":["https://openalex.org/I165733156"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lu, Jin","raw_affiliation_strings":["University of Georgia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Georgia","institution_ids":["https://openalex.org/I165733156"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007300551","display_name":"Geng Yuan","orcid":"https://orcid.org/0000-0001-9844-992X","observed_orcids":["https://orcid.org/0000-0001-9844-992X"]},"institutions":[{"id":"https://openalex.org/I165733156","display_name":"University of Georgia","ror":"https://ror.org/00te3t702","country_code":"US","type":"education","lineage":["https://openalex.org/I165733156"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuan, Geng","raw_affiliation_strings":["University of Georgia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Georgia","institution_ids":["https://openalex.org/I165733156"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.54830002784729,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.54830002784729,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.16019999980926514,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.12729999423027039,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/speculative-decoding","display_name":"speculative decoding","score":0.9660000205039978},{"id":"https://openalex.org/keywords/kv-cache","display_name":"KV cache","score":0.574999988079071},{"id":"https://openalex.org/keywords/long-context-inference","display_name":"long-context inference","score":0.8960000276565552},{"id":"https://openalex.org/keywords/autoregressive-generation","display_name":"autoregressive generation","score":0.777999997138977},{"id":"https://openalex.org/keywords/edge-devices","display_name":"edge devices","score":0.43700000643730164},{"id":"https://openalex.org/keywords/llama","display_name":"llamas","score":0.7429999709129333}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7741000056266785},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.7714999914169312},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.5490999817848206},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.5192000269889832},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.49900001287460327},{"id":"https://openalex.org/C159877910","wikidata":"https://www.wikidata.org/wiki/Q2202883","display_name":"Autoregressive model","level":2,"score":0.4683000147342682},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.44519999623298645},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4074999988079071},{"id":"https://openalex.org/C193969084","wikidata":"https://www.wikidata.org/wiki/Q7452500","display_name":"Sequential decoding","level":4,"score":0.4068000018596649},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.3637999892234802},{"id":"https://openalex.org/C117896860","wikidata":"https://www.wikidata.org/wiki/Q11376","display_name":"Acceleration","level":2,"score":0.3637000024318695},{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.3603000044822693},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3571000099182129},{"id":"https://openalex.org/C147297375","wikidata":"https://www.wikidata.org/wiki/Q6674930","display_name":"Look-ahead","level":2,"score":0.32710000872612},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.3224000036716461},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.32199999690055847},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.3197000026702881},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.3084000051021576},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.30329999327659607},{"id":"https://openalex.org/C38677869","wikidata":"https://www.wikidata.org/wiki/Q724168","display_name":"Petri net","level":2,"score":0.2948000133037567},{"id":"https://openalex.org/C2779696439","wikidata":"https://www.wikidata.org/wiki/Q7512811","display_name":"Signature (topology)","level":2,"score":0.2678999900817871},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.2669999897480011},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.26460000872612},{"id":"https://openalex.org/C12186640","wikidata":"https://www.wikidata.org/wiki/Q6815743","display_name":"Memory model","level":3,"score":0.25859999656677246},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.25380000472068787}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.02800","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.02800","pdf_url":"https://arxiv.org/pdf/2610.02800","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.02800","doi":"https://doi.org/10.48550/arxiv.2610.02800","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.02800","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.02800","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.02800","pdf_url":"https://arxiv.org/pdf/2610.02800","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7220458334.pdf","grobid_xml":"https://content.openalex.org/works/W7220458334.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7222962345","https://openalex.org/W4416034535","https://openalex.org/W4415274683","https://openalex.org/W7154656237","https://openalex.org/W7106822619","https://openalex.org/W4409164525","https://openalex.org/W4414690743","https://openalex.org/W7171963079","https://openalex.org/W4415910141","https://openalex.org/W4416868380"],"abstract_inverted_index":{"Speculative":[0],"decoding":[1,59,169],"accelerates":[2],"autoregressive":[3,150],"generation":[4],"by":[5,157],"using":[6],"a":[7,56,63,75,78,84,102],"lightweight":[8],"draft":[9,24,46,65,76],"to":[10,100,113],"propose":[11,54],"multiple":[12,121],"tokens":[13],"for":[14,117],"parallel":[15],"verification.":[16],"However,":[17],"existing":[18],"methods":[19],"often":[20],"require":[21],"an":[22,130],"additional":[23],"model":[25,141],"or":[26,167],"weight":[27,105],"representation,":[28],"introducing":[29],"non-negligible":[30],"memory":[31],"overhead":[32],"on":[33],"resource-constrained":[34],"devices.":[35],"Self-speculative":[36],"approaches":[37],"reduce":[38],"this":[39,110],"overhead,":[40],"yet":[41],"still":[42],"face":[43],"trade-offs":[44],"between":[45],"quality,":[47,49,142],"target":[48,70,93],"and":[50,88,125,143],"storage":[51],"efficiency.":[52],"We":[53],"BitNest,":[55],"bit-nested":[57],"speculative":[58,132],"framework":[60],"that":[61],"embeds":[62],"low-precision":[64,86],"directly":[66],"into":[67],"the":[68,91,114,153],"higher-precision":[69,92,140],"representation.":[71,106],"Instead":[72],"of":[73,135],"deriving":[74],"from":[77],"predefined":[79],"target,":[80],"BitNest":[81,107,128,162],"first":[82],"constructs":[83],"strong":[85],"base":[87],"then":[89],"recovers":[90],"through":[94],"residual":[95],"refinement,":[96],"enabling":[97],"both":[98],"models":[99,155],"share":[101],"single":[103],"physical":[104],"further":[108],"extends":[109],"progressive-precision":[111],"design":[112],"KV":[115],"cache":[116],"long-context":[118],"inference.":[119],"Across":[120],"7B--8B":[122],"edge-friendly":[123],"LLMs":[124],"diverse":[126],"workloads,":[127],"achieves":[129,164],"average":[131],"acceptance":[133],"rate":[134],"95.2%":[136],"while":[137],"closely":[138],"preserving":[139],"delivers":[144],"1.48--1.61x":[145],"end-to-end":[146],"speedup":[147],"over":[148],"FP16":[149],"decoding.":[151],"On":[152],"LLaMA":[154],"supported":[156],"all":[158],"representative":[159],"self-speculative":[160],"baselines,":[161],"also":[163],"consistently":[165],"competitive":[166],"higher":[168],"speedup.":[170]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-10-06T00:00:00"}