{"id":"https://openalex.org/W4411486604","doi":"https://doi.org/10.1145/3695053.3730996","title":"SpecEE: Accelerating Large Language Model Inference with Speculative Early Exiting","display_name":"SpecEE: Accelerating Large Language Model Inference with Speculative Early Exiting","publication_year":2025,"publication_date":"2025-06-20","ids":{"openalex":"https://openalex.org/W4411486604","doi":"https://doi.org/10.1145/3695053.3730996"},"language":"en","primary_location":{"id":"doi:10.1145/3695053.3730996","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3695053.3730996","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3695053.3730996","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 52nd Annual International Symposium on Computer Architecture","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3695053.3730996","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111878216","display_name":"Jianrong Xu","orcid":"https://orcid.org/0009-0000-7000-6537"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiaming Xu","raw_affiliation_strings":["Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0000-7000-6537","affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103512739","display_name":"Jiayi Pan","orcid":null},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiayi Pan","raw_affiliation_strings":["Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0007-0468-4592","affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yongkang Zhou","orcid":"https://orcid.org/0009-0008-7732-6347"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yongkang Zhou","raw_affiliation_strings":["Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0008-7732-6347","affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078363092","display_name":"S. Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Siming Chen","raw_affiliation_strings":["Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0004-5931-7827","affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068894182","display_name":"Jinhao Li","orcid":"https://orcid.org/0009-0009-4286-6359"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jinhao Li","raw_affiliation_strings":["Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0009-4286-6359","affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102585583","display_name":"Yaoxiu Lian","orcid":"https://orcid.org/0009-0007-7858-5132"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yaoxiu Lian","raw_affiliation_strings":["Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0007-7858-5132","affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101718586","display_name":"Junyi Wu","orcid":"https://orcid.org/0009-0008-3437-2087"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junyi Wu","raw_affiliation_strings":["Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0008-3437-2087","affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015946486","display_name":"Guohao Dai","orcid":"https://orcid.org/0000-0003-0849-3252"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guohao Dai","raw_affiliation_strings":["Shanghai Jiao Tong University; Infinigence-AI; SII, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-0849-3252","affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University; Infinigence-AI; SII, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I183067930"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"467","last_page":"481"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9882000088691711,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.779929518699646},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.580924928188324},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4927063286304474},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2201056182384491}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.779929518699646},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.580924928188324},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4927063286304474},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2201056182384491}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3695053.3730996","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3695053.3730996","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3695053.3730996","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 52nd Annual International Symposium on Computer Architecture","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3695053.3730996","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3695053.3730996","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3695053.3730996","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 52nd Annual International Symposium on Computer Architecture","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.5600000023841858,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G595567771","display_name":null,"funder_award_id":"62104128, U21B2031","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6121003335","display_name":null,"funder_award_id":"24QB2706200","funder_id":"https://openalex.org/F4320327803","funder_display_name":"Shanghai Rising-Star Program"},{"id":"https://openalex.org/G6803541034","display_name":null,"funder_award_id":"U21B2031","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8262591375","display_name":null,"funder_award_id":"62104128","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320327803","display_name":"Shanghai Rising-Star Program","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4411486604.pdf","grobid_xml":"https://content.openalex.org/works/W4411486604.grobid-xml"},"referenced_works_count":13,"referenced_works":["https://openalex.org/W1481588857","https://openalex.org/W1990571122","https://openalex.org/W2008056655","https://openalex.org/W2040870580","https://openalex.org/W2103318667","https://openalex.org/W2251939518","https://openalex.org/W2798515322","https://openalex.org/W2912924812","https://openalex.org/W2963929190","https://openalex.org/W2979826702","https://openalex.org/W3168652588","https://openalex.org/W4321636575","https://openalex.org/W4387321091"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Early":[0],"exiting":[1,41,118,168],"has":[2],"recently":[3],"emerged":[4],"as":[5,33,158],"a":[6,63],"promising":[7],"technique":[8],"for":[9,79],"accelerating":[10],"large":[11],"language":[12],"models":[13,68,93],"(LLMs)":[14],"by":[15],"effectively":[16],"reducing":[17],"the":[18,29,34,39,46,121,129,150,165,179],"hardware":[19],"computation":[20],"and":[21,43,102,175],"memory":[22],"access.In":[23],"this":[24,71],"paper,":[25],"we":[26],"identify":[27],"that":[28,97],"LLM":[30],"vocabulary":[31,58],"serves":[32],"runtime":[35],"search":[36,72,161],"space":[37,162],"of":[38,113,132,143,153],"early":[40,117,167],"predictor":[42,47,81,84,89,115,122,144],"significantly":[44],"influences":[45],"workload":[48],"(e.g.,":[49],"20%":[50,137],"overall":[51,109],"inference":[52,110,138],"latency":[53],"with":[54,85,94],"3":[55],"10":[56],"4":[57],"size":[59],"in":[60,107,123,145,149],"Llama2).We":[61],"propose":[62],"novel":[64],"paradigm":[65],"using":[66],"speculative":[67,146,154],"to":[69,136,171,177],"reduce":[70],"space,":[73],"while":[74],"addressing":[75],"three":[76],"critical":[77],"challenges":[78],"further":[80],"optimization.(1)":[82],"Time-consuming":[83],"high":[86],"computational":[87],"complexity.Current":[88],"designs":[90],"leverage":[91],"basic":[92],"high-dimensional":[95],"input":[96],"ignore":[98],"inherent":[99],"data":[100],"variation":[101],"GPU":[103],"parallelization":[104],"opportunities,":[105],"resulting":[106],"15%":[108],"latency.(2)":[111],"Under-utilization":[112],"layer-wise":[114,133],"deployment.Current":[116],"systems":[119],"treat":[120],"each":[124],"layer":[125],"equally":[126],"without":[127],"considering":[128],"activation":[130],"frequencies":[131],"predictors,":[134],"leading":[135,170],"overhead.(3)":[139],"Exponential":[140],"mapping":[141,173],"complexity":[142,174],"decoding.Each":[147],"token":[148,151],"tree":[152],"decoding":[155],"is":[156],"treated":[157],"an":[159],"independent":[160],"when":[163],"applying":[164],"current":[166],"mapping,":[169],"exponential":[172],"failing":[176],"incorporate":[178],"high-throughput":[180],"benefits":[181]},"counts_by_year":[{"year":2026,"cited_by_count":5}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}