{"id":"https://openalex.org/W4407186969","doi":"https://doi.org/10.52202/085713-5307","title":"EasySpec: Layer-Parallel Speculative Decoding for Efficient Multi-GPU Utilization","display_name":"EasySpec: Layer-Parallel Speculative Decoding for Efficient Multi-GPU Utilization","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4407186969","doi":"https://doi.org/10.52202/085713-5307"},"language":"en","primary_location":{"id":"doi:10.52202/085713-5307","doi":"https://doi.org/10.52202/085713-5307","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-5307","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.02493","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102016612","display_name":"Yize Wu","orcid":"https://orcid.org/0009-0001-8202-5556","observed_orcids":["https://orcid.org/0009-0001-8202-5556"]},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yize Wu","raw_affiliation_strings":["Institute of Software, Chinese Academy of Sciences"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Software, Chinese Academy of Sciences","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}]},{"author_position":"middle","author":{"id":null,"display_name":"KE GAO","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"KE GAO","raw_affiliation_strings":["Chinese Academy of Sciences"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Chinese Academy of Sciences","institution_ids":["https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Ling Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ling Li","raw_affiliation_strings":["Institute of Software, CAS"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Software, CAS","institution_ids":["https://openalex.org/I4210128818"]}]},{"author_position":"last","author":{"id":null,"display_name":"Yanjun Wu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanjun Wu","raw_affiliation_strings":["Institute of Software, Chinese Academy of Sciences"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Software, Chinese Academy of Sciences","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.3567,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.54602513,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"176367","last_page":"176391"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.7310000061988831,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.7310000061988831,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.10130000114440918,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.09149999916553497,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/speculative-decoding","display_name":"speculative decoding","score":0.9800000190734863},{"id":"https://openalex.org/keywords/large-language-model-inference","display_name":"large language model inference","score":0.7509999871253967},{"id":"https://openalex.org/keywords/tensor-parallelism","display_name":"tensor parallelism","score":0.9449999928474426},{"id":"https://openalex.org/keywords/draft-model","display_name":"draft model","score":0.8159999847412109},{"id":"https://openalex.org/keywords/inference-latency-reduction","display_name":"inference latency reduction","score":0.6809999942779541},{"id":"https://openalex.org/keywords/gpu-utilization","display_name":"GPU utilization","score":0.5879999995231628},{"id":"https://openalex.org/keywords/training-free-acceleration","display_name":"training-free acceleration","score":0.5659999847412109}],"concepts":[{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.7146685123443604},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6697477698326111},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.6471582651138306},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.5836552977561951},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.09231576323509216},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.09042206406593323},{"id":"https://openalex.org/C171250308","wikidata":"https://www.wikidata.org/wiki/Q11468","display_name":"Nanotechnology","level":1,"score":0.05981984734535217}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.52202/085713-5307","doi":"https://doi.org/10.52202/085713-5307","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-5307","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2502.02493","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.02493","pdf_url":"https://arxiv.org/pdf/2502.02493","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.02493","doi":"https://doi.org/10.48550/arxiv.2502.02493","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.02493","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.02493","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.02493","pdf_url":"https://arxiv.org/pdf/2502.02493","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G6748911986","display_name":null,"funder_award_id":"92364202","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4407186969.pdf","grobid_xml":"https://content.openalex.org/works/W4407186969.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W1821462560","https://openalex.org/W2973727699","https://openalex.org/W3083410900","https://openalex.org/W3134642945","https://openalex.org/W3177813494","https://openalex.org/W4303648904","https://openalex.org/W4319166707","https://openalex.org/W4327810158","https://openalex.org/W4380714727","https://openalex.org/W4388718089","https://openalex.org/W4389326242","https://openalex.org/W4391124766","https://openalex.org/W4391335474","https://openalex.org/W4401042997","https://openalex.org/W4401306886","https://openalex.org/W4401660192","https://openalex.org/W4402672007","https://openalex.org/W4402683901","https://openalex.org/W4403796339","https://openalex.org/W4404782770","https://openalex.org/W4405655184"],"related_works":["https://openalex.org/W4403346564","https://openalex.org/W4416243492","https://openalex.org/W7152101543","https://openalex.org/W7160652145","https://openalex.org/W7214784545","https://openalex.org/W4415798602","https://openalex.org/W4415274683","https://openalex.org/W4415153626","https://openalex.org/W7155084245","https://openalex.org/W4388092893"],"abstract_inverted_index":{"Speculative":[0],"decoding":[1],"is":[2,27,57,87,139,155,227],"an":[3],"effective":[4],"and":[5,158],"lossless":[6],"method":[7],"for":[8],"Large":[9],"Language":[10],"Model":[11],"(LLM)":[12],"inference":[13,38],"acceleration.":[14],"It":[15],"employs":[16],"a":[17,22,97,142,156,187,217],"smaller":[18,59,170],"model":[19,56],"to":[20,67,121,193,214],"generate":[21],"draft":[23,55,116,135],"token":[24],"sequence,":[25],"which":[26,86],"then":[28],"verified":[29],"by":[30,212],"the":[31,49,54,63,71,81,103,110,115,134,175,198,202,206],"original":[32,199],"base":[33,64,203],"model.":[34],"In":[35],"multi-GPU":[36,106],"systems,":[37],"latency":[39],"can":[40,185,209],"be":[41,210],"further":[42],"reduced":[43],"through":[44],"tensor":[45],"parallelism":[46],"(TP),":[47],"while":[48,196],"optimal":[50],"TP":[51],"size":[52],"of":[53,62,84,105,172,190,201,222],"typically":[58],"than":[60],"that":[61,76,101,183],"model,":[65,117],"leading":[66],"GPU":[68],"idling":[69],"during":[70],"drafting":[72,207],"stage.":[73],"We":[74,161],"observe":[75],"such":[77],"inefficiency":[78],"stems":[79],"from":[80,174],"sequential":[82],"execution":[83],"layers,":[85],"seemingly":[88],"natural":[89],"but":[90],"actually":[91],"unnecessary.":[92],"Therefore,":[93],"we":[94],"propose":[95],"EasySpec,":[96],"layer-parallel":[98],"speculation":[99,219],"strategy":[100],"optimizes":[102],"efficiency":[104],"utilization.":[107],"EasySpec":[108,154,163,184],"breaks":[109],"inter-layer":[111],"data":[112],"dependencies":[113],"in":[114,141],"enabling":[118],"multiple":[119,125],"layers":[120],"run":[122],"simultaneously":[123],"across":[124],"devices":[126],"as":[127,178],"'fuzzy'":[128],"speculation.":[129],"After":[130],"each":[131],"drafting-and-verification":[132],"iteration,":[133],"model's":[136],"key-value":[137],"cache":[138],"calibrated":[140],"single":[143],"forward":[144],"pass,":[145],"preventing":[146],"long-term":[147],"fuzzy-error":[148],"accumulation":[149],"at":[150,229],"minimal":[151],"additional":[152],"latency.":[153],"training-free":[157],"plug-in":[159],"method.":[160],"evaluated":[162],"on":[164],"several":[165],"mainstream":[166],"open-source":[167],"LLMs,":[168],"using":[169],"versions":[171],"models":[173],"same":[176],"series":[177],"drafters.":[179],"The":[180,225],"results":[181],"demonstrate":[182],"achieve":[186],"peak":[188],"speedup":[189],"4.17x":[191],"compared":[192],"vanilla":[194],"decoding,":[195],"preserving":[197],"distributions":[200],"LLMs.":[204],"Specifically,":[205],"stage":[208],"accelerated":[211],"up":[213],"1.62x":[215],"with":[216],"maximum":[218],"accuracy":[220],"drop":[221],"only":[223],"7%.":[224],"code":[226],"available":[228],"https://github.com/Yize-Wu/EasySpec.":[230]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-02-06T00:00:00"}