{"id":"https://openalex.org/W4395112660","doi":"https://doi.org/10.1145/3620666.3651335","title":"SpecInfer: Accelerating Large Language Model Serving with Tree-based Speculative Inference and Verification","display_name":"SpecInfer: Accelerating Large Language Model Serving with Tree-based Speculative Inference and Verification","publication_year":2024,"publication_date":"2024-04-24","ids":{"openalex":"https://openalex.org/W4395112660","doi":"https://doi.org/10.1145/3620666.3651335"},"language":"en","primary_location":{"id":"doi:10.1145/3620666.3651335","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3620666.3651335","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 3","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3620666.3651335","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015552951","display_name":"Xupeng Miao","orcid":"https://orcid.org/0000-0002-9371-8358"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":true,"raw_author_name":"Xupeng Miao","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA","Peking University Beijing, China","Tsinghua University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Carnegie Mellon University Pittsburgh, PA, USA","University of California, San Diego San Diego, CA, USA","Stanford University Stanford, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-9371-8358","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073183750","display_name":"Gabriele Oliaro","orcid":"https://orcid.org/0000-0001-5406-0736"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Gabriele Oliaro","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Stanford University Stanford, CA, USA","Shanghai Jiao Tong University Shanghai, China","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA","Peking University Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-5406-0736","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072348433","display_name":"Zhihao Zhang","orcid":"https://orcid.org/0009-0002-8409-2717"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Zhihao Zhang","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Tsinghua University, Beijing, China","Peking University Beijing, China","Stanford University Stanford, CA, USA","University of California, San Diego San Diego, CA, USA","Shanghai Jiao Tong University Shanghai, China","Tsinghua University Beijing, China"],"raw_orcid":"https://orcid.org/0009-0002-8409-2717","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040267796","display_name":"Xinhao Cheng","orcid":"https://orcid.org/0009-0009-3375-497X"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Xinhao Cheng","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Stanford University Stanford, CA, USA","Shanghai Jiao Tong University Shanghai, China","University of California, San Diego San Diego, CA, USA","Tsinghua University Beijing, China","Peking University Beijing, China"],"raw_orcid":"https://orcid.org/0009-0009-3375-497X","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006081199","display_name":"Zeyu Wang","orcid":"https://orcid.org/0009-0002-5756-2744"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Zeyu Wang","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","Peking University Beijing, China","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0002-5756-2744","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076465867","display_name":"Zhengxin Zhang","orcid":"https://orcid.org/0000-0003-1578-2597"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Zhengxin Zhang","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Tsinghua University, Beijing, China","Carnegie Mellon University Pittsburgh, PA, USA","Peking University Beijing, China","Stanford University Stanford, CA, USA","University of California, San Diego San Diego, CA, USA","Shanghai Jiao Tong University Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-1578-2597","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101882411","display_name":"Rae Ying Yee Wong","orcid":"https://orcid.org/0009-0008-2677-0659"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Rae Ying Yee Wong","raw_affiliation_strings":["Stanford University, Stanford, USA","Tsinghua University Beijing, China","Peking University Beijing, China","Stanford University Stanford, CA, USA","Shanghai Jiao Tong University Shanghai, China","University of California, San Diego San Diego, CA, USA","Carnegie Mellon University Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0009-0008-2677-0659","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007886144","display_name":"Alan Zhu","orcid":"https://orcid.org/0009-0001-8694-9246"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Alan Zhu","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA","Shanghai Jiao Tong University Shanghai, China","Carnegie Mellon University Pittsburgh, PA, USA","Stanford University Stanford, CA, USA","Peking University Beijing, China"],"raw_orcid":"https://orcid.org/0009-0001-8694-9246","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103135377","display_name":"Lijie Yang","orcid":"https://orcid.org/0009-0004-7909-5416"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Lijie Yang","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Tsinghua University Beijing, China","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0004-7909-5416","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054091574","display_name":"Xiaoxiang Shi","orcid":"https://orcid.org/0009-0000-6840-4691"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Xiaoxiang Shi","raw_affiliation_strings":["Shanghai Jiao Tong University, Shanghai, China","Stanford University Stanford, CA, USA","University of California, San Diego San Diego, CA, USA","Peking University Beijing, China","Tsinghua University Beijing, China","Carnegie Mellon University Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0009-0000-6840-4691","affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010483833","display_name":"Chunan Shi","orcid":"https://orcid.org/0009-0009-7197-4965"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Chunan Shi","raw_affiliation_strings":["Peking University, Beijing, China","Stanford University Stanford, CA, USA","Shanghai Jiao Tong University Shanghai, China","University of California, San Diego San Diego, CA, USA","Tsinghua University Beijing, China","Carnegie Mellon University Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0009-0009-7197-4965","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018889907","display_name":"Zhuoming Chen","orcid":"https://orcid.org/0009-0006-7797-573X"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Zhuoming Chen","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA","Tsinghua University Beijing, China","Peking University Beijing, China","Carnegie Mellon University Pittsburgh, PA, USA","University of California, San Diego San Diego, CA, USA","Stanford University Stanford, CA, USA","Shanghai Jiao Tong University Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0006-7797-573X","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027897868","display_name":"Daiyaan Arfeen","orcid":"https://orcid.org/0009-0009-5626-4551"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Daiyaan Arfeen","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Shanghai Jiao Tong University Shanghai, China","Peking University Beijing, China","Stanford University Stanford, CA, USA","University of California, San Diego San Diego, CA, USA","Tsinghua University Beijing, China"],"raw_orcid":"https://orcid.org/0009-0009-5626-4551","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082159350","display_name":"Reyna Abhyankar","orcid":"https://orcid.org/0009-0005-6763-0108"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Reyna Abhyankar","raw_affiliation_strings":["University of California San Diego, San Diego, USA","University of California, San Diego San Diego, CA, USA","Carnegie Mellon University Pittsburgh, PA, USA","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0005-6763-0108","affiliations":[{"raw_affiliation_string":"University of California San Diego, San Diego, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062437461","display_name":"Zhihao Jia","orcid":"https://orcid.org/0000-0002-1270-5185"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Zhihao Jia","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","University of California, San Diego San Diego, CA, USA","Tsinghua University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Peking University Beijing, China","Stanford University Stanford, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-1270-5185","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":15,"corresponding_author_ids":["https://openalex.org/A5015552951"],"corresponding_institution_ids":["https://openalex.org/I183067930","https://openalex.org/I20231570","https://openalex.org/I36258959","https://openalex.org/I74973139","https://openalex.org/I97018004","https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":22.4444,"has_fulltext":false,"cited_by_count":73,"citation_normalized_percentile":{"value":0.99607643,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"932","last_page":"949"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9835000038146973,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.809113085269928},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.58620685338974},{"id":"https://openalex.org/keywords/tree","display_name":"Tree (set theory)","score":0.43485501408576965},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4085949659347534},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.33851349353790283},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2950827479362488},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09067410230636597}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.809113085269928},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.58620685338974},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.43485501408576965},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4085949659347534},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33851349353790283},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2950827479362488},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09067410230636597},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3620666.3651335","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3620666.3651335","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 3","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3620666.3651335","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3620666.3651335","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 3","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W1516184288","https://openalex.org/W1555915743","https://openalex.org/W2145073242","https://openalex.org/W2158992088","https://openalex.org/W2252136820","https://openalex.org/W2725179571","https://openalex.org/W2804032941","https://openalex.org/W2884700152","https://openalex.org/W2890152612","https://openalex.org/W2973727699","https://openalex.org/W2981758446","https://openalex.org/W2995259046","https://openalex.org/W2998617917","https://openalex.org/W3030163527","https://openalex.org/W3132669550","https://openalex.org/W3149839747","https://openalex.org/W3159727696","https://openalex.org/W3173154111","https://openalex.org/W4200634402","https://openalex.org/W4287327026","https://openalex.org/W4289302788","https://openalex.org/W4292779060","https://openalex.org/W4313484599","https://openalex.org/W4324297016","https://openalex.org/W4385245566","https://openalex.org/W6607397737","https://openalex.org/W6681651645","https://openalex.org/W6739901393","https://openalex.org/W7000926438"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W4395014643","https://openalex.org/W4391913857","https://openalex.org/W2350741829"],"abstract_inverted_index":{"This":[0],"paper":[1],"introduces":[2],"SpecInfer,":[3],"a":[4,40,47,60,71,82],"system":[5],"that":[6,112],"accelerates":[7],"generative":[8,102,136],"large":[9],"language":[10],"model":[11,107],"(LLM)":[12],"serving":[13,101,117],"with":[14],"tree-based":[15,73],"speculative":[16,28],"inference":[17,124],"and":[18,97,125],"verification.":[19],"The":[20,51],"key":[21],"idea":[22],"behind":[23],"SpecInfer":[24,77,113,138],"is":[25,63,139],"leveraging":[26],"small":[27],"models":[29],"to":[30],"predict":[31],"the":[32,35,66,94,134],"LLM's":[33],"outputs;":[34],"predictions":[36],"are":[37],"organized":[38],"as":[39,81],"token":[41,49,56,61,83],"tree,":[42],"whose":[43],"nodes":[44],"each":[45],"represent":[46],"candidate":[48,55],"sequence.":[50],"correctness":[52],"of":[53,87],"all":[54],"sequences":[57],"represented":[58],"by":[59,119,126],"tree":[62,84],"verified":[64],"against":[65],"LLM":[67,80,116,123,130],"in":[68],"parallel":[69,74],"using":[70],"novel":[72],"decoding":[75],"mechanism.":[76],"uses":[78],"an":[79,88],"verifier":[85],"instead":[86],"incremental":[89],"decoder,":[90],"which":[91],"significantly":[92],"reduces":[93],"end-to-end":[95],"latency":[96],"computational":[98],"requirement":[99],"for":[100,121,128],"LLMs":[103],"while":[104,132],"provably":[105],"preserving":[106,133],"quality.":[108],"Our":[109],"evaluation":[110],"shows":[111],"outperforms":[114],"existing":[115],"systems":[118],"1.5-2.8\u00d7":[120],"distributed":[122],"2.6-3.5\u00d7":[127],"offloading-based":[129],"inference,":[131],"same":[135],"performance.":[137],"publicly":[140],"available":[141],"at":[142],"https://github.com/flexflow/FlexFlow/":[143]},"counts_by_year":[{"year":2026,"cited_by_count":18},{"year":2025,"cited_by_count":47},{"year":2024,"cited_by_count":8}],"updated_date":"2026-06-08T08:47:23.578185","created_date":"2024-04-25T00:00:00"}