{"id":"https://openalex.org/W4395112660","doi":"https://doi.org/10.1145/3620666.3651335","title":"SpecInfer: Accelerating Large Language Model Serving with Tree-based Speculative Inference and Verification","display_name":"SpecInfer: Accelerating Large Language Model Serving with Tree-based Speculative Inference and Verification","publication_year":2024,"publication_date":"2024-04-24","ids":{"openalex":"https://openalex.org/W4395112660","doi":"https://doi.org/10.1145/3620666.3651335"},"language":"en","primary_location":{"id":"doi:10.1145/3620666.3651335","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3620666.3651335","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 3","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3620666.3651335","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015552951","display_name":"Xupeng Miao","orcid":"https://orcid.org/0000-0002-9371-8358"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Xupeng Miao","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA","Carnegie Mellon University Pittsburgh, PA, USA","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-9371-8358","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073183750","display_name":"Gabriele Oliaro","orcid":"https://orcid.org/0000-0001-5406-0736"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Gabriele Oliaro","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0000-0001-5406-0736","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072348433","display_name":"Zhihao Zhang","orcid":"https://orcid.org/0009-0002-8409-2717"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Zhihao Zhang","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Tsinghua University, Beijing, China","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0002-8409-2717","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040267796","display_name":"Xinhao Cheng","orcid":"https://orcid.org/0009-0009-3375-497X"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Xinhao Cheng","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0009-3375-497X","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006081199","display_name":"Zeyu Wang","orcid":"https://orcid.org/0009-0002-5756-2744"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Zeyu Wang","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0002-5756-2744","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076465867","display_name":"Zhengxin Zhang","orcid":"https://orcid.org/0000-0003-1578-2597"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Zhengxin Zhang","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Tsinghua University, Beijing, China","Carnegie Mellon University Pittsburgh, PA, USA","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0000-0003-1578-2597","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101882411","display_name":"Rae Ying Yee Wong","orcid":"https://orcid.org/0009-0008-2677-0659"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Rae Ying Yee Wong","raw_affiliation_strings":["Stanford University, Stanford, USA","Carnegie Mellon University Pittsburgh, PA, USA","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0008-2677-0659","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007886144","display_name":"Alan Zhu","orcid":"https://orcid.org/0009-0001-8694-9246"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Alan Zhu","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA","Carnegie Mellon University Pittsburgh, PA, USA","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0001-8694-9246","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103135377","display_name":"Lijie Yang","orcid":"https://orcid.org/0009-0004-7909-5416"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Lijie Yang","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0004-7909-5416","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054091574","display_name":"Xiaoxiang Shi","orcid":"https://orcid.org/0009-0000-6840-4691"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Xiaoxiang Shi","raw_affiliation_strings":["Shanghai Jiao Tong University, Shanghai, China","Carnegie Mellon University Pittsburgh, PA, USA","Peking University Beijing, China","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0000-6840-4691","affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010483833","display_name":"Chunan Shi","orcid":"https://orcid.org/0009-0009-7197-4965"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Chunan Shi","raw_affiliation_strings":["Peking University, Beijing, China","Carnegie Mellon University Pittsburgh, PA, USA","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0009-7197-4965","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018889907","display_name":"Zhuoming Chen","orcid":"https://orcid.org/0009-0006-7797-573X"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Zhuoming Chen","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA","Carnegie Mellon University Pittsburgh, PA, USA","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0006-7797-573X","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027897868","display_name":"Daiyaan Arfeen","orcid":"https://orcid.org/0009-0009-5626-4551"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Daiyaan Arfeen","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0009-5626-4551","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082159350","display_name":"Reyna Abhyankar","orcid":"https://orcid.org/0009-0005-6763-0108"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Reyna Abhyankar","raw_affiliation_strings":["University of California San Diego, San Diego, USA","Carnegie Mellon University Pittsburgh, PA, USA","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0009-0005-6763-0108","affiliations":[{"raw_affiliation_string":"University of California San Diego, San Diego, USA","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"Carnegie Mellon University Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062437461","display_name":"Zhihao Jia","orcid":"https://orcid.org/0000-0002-1270-5185"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Zhihao Jia","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States of America","Peking University Beijing, China","Shanghai Jiao Tong University Shanghai, China","Stanford University Stanford, CA, USA","Tsinghua University Beijing, China","University of California, San Diego San Diego, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-1270-5185","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States of America","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Peking University Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Shanghai Jiao Tong University Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Stanford University Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Tsinghua University Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"University of California, San Diego San Diego, CA, USA","institution_ids":["https://openalex.org/I36258959"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":28.8034,"has_fulltext":false,"cited_by_count":81,"citation_normalized_percentile":{"value":0.99799491,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"932","last_page":"949"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9835000038146973,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.809113085269928},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.58620685338974},{"id":"https://openalex.org/keywords/tree","display_name":"Tree (set theory)","score":0.43485501408576965},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4085949659347534},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.33851349353790283},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2950827479362488},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09067410230636597}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.809113085269928},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.58620685338974},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.43485501408576965},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4085949659347534},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33851349353790283},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2950827479362488},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09067410230636597},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3620666.3651335","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3620666.3651335","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 3","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3620666.3651335","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3620666.3651335","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 3","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W1516184288","https://openalex.org/W1555915743","https://openalex.org/W2145073242","https://openalex.org/W2158992088","https://openalex.org/W2252136820","https://openalex.org/W2725179571","https://openalex.org/W2804032941","https://openalex.org/W2884700152","https://openalex.org/W2890152612","https://openalex.org/W2973727699","https://openalex.org/W2981758446","https://openalex.org/W2995259046","https://openalex.org/W2998617917","https://openalex.org/W3030163527","https://openalex.org/W3132669550","https://openalex.org/W3149839747","https://openalex.org/W3159727696","https://openalex.org/W3173154111","https://openalex.org/W4200634402","https://openalex.org/W4287327026","https://openalex.org/W4289302788","https://openalex.org/W4292779060","https://openalex.org/W4313484599","https://openalex.org/W4324297016","https://openalex.org/W4385245566","https://openalex.org/W6607397737","https://openalex.org/W6681651645","https://openalex.org/W6739901393","https://openalex.org/W7000926438"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W4395014643","https://openalex.org/W4391913857","https://openalex.org/W2350741829"],"abstract_inverted_index":{"This":[0],"paper":[1],"introduces":[2],"SpecInfer,":[3],"a":[4,40,47,60,71,82],"system":[5],"that":[6,112],"accelerates":[7],"generative":[8,102,136],"large":[9],"language":[10],"model":[11,107],"(LLM)":[12],"serving":[13,101,117],"with":[14],"tree-based":[15,73],"speculative":[16,28],"inference":[17,124],"and":[18,97,125],"verification.":[19],"The":[20,51],"key":[21],"idea":[22],"behind":[23],"SpecInfer":[24,77,113,138],"is":[25,63,139],"leveraging":[26],"small":[27],"models":[29],"to":[30],"predict":[31],"the":[32,35,66,94,134],"LLM's":[33],"outputs;":[34],"predictions":[36],"are":[37],"organized":[38],"as":[39,81],"token":[41,49,56,61,83],"tree,":[42],"whose":[43],"nodes":[44],"each":[45],"represent":[46],"candidate":[48,55],"sequence.":[50],"correctness":[52],"of":[53,87],"all":[54],"sequences":[57],"represented":[58],"by":[59,119,126],"tree":[62,84],"verified":[64],"against":[65],"LLM":[67,80,116,123,130],"in":[68],"parallel":[69,74],"using":[70],"novel":[72],"decoding":[75],"mechanism.":[76],"uses":[78],"an":[79,88],"verifier":[85],"instead":[86],"incremental":[89],"decoder,":[90],"which":[91],"significantly":[92],"reduces":[93],"end-to-end":[95],"latency":[96],"computational":[98],"requirement":[99],"for":[100,121,128],"LLMs":[103],"while":[104,132],"provably":[105],"preserving":[106,133],"quality.":[108],"Our":[109],"evaluation":[110],"shows":[111],"outperforms":[114],"existing":[115],"systems":[118],"1.5-2.8\u00d7":[120],"distributed":[122],"2.6-3.5\u00d7":[127],"offloading-based":[129],"inference,":[131],"same":[135],"performance.":[137],"publicly":[140],"available":[141],"at":[142],"https://github.com/flexflow/FlexFlow/":[143]},"counts_by_year":[{"year":2026,"cited_by_count":25},{"year":2025,"cited_by_count":48},{"year":2024,"cited_by_count":8}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2024-04-25T00:00:00"}