{"id":"https://openalex.org/W7167071630","doi":"https://doi.org/10.1145/3797905.3800530","title":"InferFast: Bridging the Gap Between Unstructured LLM Sparsity and Practical GPU Throughput","display_name":"InferFast: Bridging the Gap Between Unstructured LLM Sparsity and Practical GPU Throughput","publication_year":2026,"publication_date":"2026-07-02","ids":{"openalex":"https://openalex.org/W7167071630","doi":"https://doi.org/10.1145/3797905.3800530"},"language":null,"primary_location":{"id":"doi:10.1145/3797905.3800530","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3797905.3800530","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th ACM International Conference on Supercomputing","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3797905.3800530","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5051253135","display_name":"Zijie Shen","orcid":"https://orcid.org/0009-0005-3009-7733"},"institutions":[{"id":"https://openalex.org/I149594827","display_name":"Xidian University","ror":"https://ror.org/05s92vm98","country_code":"CN","type":"education","lineage":["https://openalex.org/I149594827"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zijie Shen","raw_affiliation_strings":["Guangzhou Institute of Technology, Xidian University, Guangzhou, China"],"raw_orcid":"https://orcid.org/0009-0008-9886-905X","affiliations":[{"raw_affiliation_string":"Guangzhou Institute of Technology, Xidian University, Guangzhou, China","institution_ids":["https://openalex.org/I149594827"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139888444","display_name":"Weifeng Bu","orcid":"https://orcid.org/0009-0000-3676-0210"},"institutions":[{"id":"https://openalex.org/I149594827","display_name":"Xidian University","ror":"https://ror.org/05s92vm98","country_code":"CN","type":"education","lineage":["https://openalex.org/I149594827"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weifeng Bu","raw_affiliation_strings":["Guangzhou Institute of Technology, Xidian University, Guangzhou, China"],"raw_orcid":"https://orcid.org/0009-0000-3676-0210","affiliations":[{"raw_affiliation_string":"Guangzhou Institute of Technology, Xidian University, Guangzhou, China","institution_ids":["https://openalex.org/I149594827"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056601955","display_name":"Kai Sheng","orcid":"https://orcid.org/0000-0001-6602-7212"},"institutions":[{"id":"https://openalex.org/I149594827","display_name":"Xidian University","ror":"https://ror.org/05s92vm98","country_code":"CN","type":"education","lineage":["https://openalex.org/I149594827"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kai Sheng","raw_affiliation_strings":["Guangzhou Institute of Technology, Xidian University, Guangzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-6602-7212","affiliations":[{"raw_affiliation_string":"Guangzhou Institute of Technology, Xidian University, Guangzhou, China","institution_ids":["https://openalex.org/I149594827"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100353508","display_name":"Hao Chen","orcid":"https://orcid.org/0000-0001-9857-6283"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hao Chen","raw_affiliation_strings":["College of Computer Science and Electronic Engineering, Hunan University, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0001-9857-6283","affiliations":[{"raw_affiliation_string":"College of Computer Science and Electronic Engineering, Hunan University, Changsha, China","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5133776688","display_name":"Xin He","orcid":"https://orcid.org/0009-0002-1481-3179"},"institutions":[{"id":"https://openalex.org/I149594827","display_name":"Xidian University","ror":"https://ror.org/05s92vm98","country_code":"CN","type":"education","lineage":["https://openalex.org/I149594827"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin He","raw_affiliation_strings":["Guangzhou Institute of Technology, Xidian University, Guangzhou, China"],"raw_orcid":"https://orcid.org/0009-0002-1481-3179","affiliations":[{"raw_affiliation_string":"Guangzhou Institute of Technology, Xidian University, Guangzhou, China","institution_ids":["https://openalex.org/I149594827"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"502","last_page":"514"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.14800000190734863,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.14800000190734863,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.10490000247955322,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.07829999923706055,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bitmap","display_name":"Bitmap","score":0.6182000041007996},{"id":"https://openalex.org/keywords/bridging","display_name":"Bridging (networking)","score":0.607699990272522},{"id":"https://openalex.org/keywords/sparse-matrix","display_name":"Sparse matrix","score":0.5889999866485596},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.550000011920929},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5404000282287598},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.5156000256538391},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.4722999930381775},{"id":"https://openalex.org/keywords/multi-core-processor","display_name":"Multi-core processor","score":0.4661000072956085},{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.4165000021457672},{"id":"https://openalex.org/keywords/memory-footprint","display_name":"Memory footprint","score":0.38999998569488525}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8440999984741211},{"id":"https://openalex.org/C3115412","wikidata":"https://www.wikidata.org/wiki/Q1194708","display_name":"Bitmap","level":2,"score":0.6182000041007996},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.607699990272522},{"id":"https://openalex.org/C56372850","wikidata":"https://www.wikidata.org/wiki/Q1050404","display_name":"Sparse matrix","level":3,"score":0.5889999866485596},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5838000178337097},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.550000011920929},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5404000282287598},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.5156000256538391},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.4722999930381775},{"id":"https://openalex.org/C78766204","wikidata":"https://www.wikidata.org/wiki/Q555032","display_name":"Multi-core processor","level":2,"score":0.4661000072956085},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.43619999289512634},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.4165000021457672},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.4025000035762787},{"id":"https://openalex.org/C74912251","wikidata":"https://www.wikidata.org/wiki/Q6815727","display_name":"Memory footprint","level":2,"score":0.38999998569488525},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.3853999972343445},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.3652999997138977},{"id":"https://openalex.org/C162319229","wikidata":"https://www.wikidata.org/wiki/Q175263","display_name":"Data structure","level":2,"score":0.36309999227523804},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.3628000020980835},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.33980000019073486},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.33970001339912415},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.30640000104904175},{"id":"https://openalex.org/C42935608","wikidata":"https://www.wikidata.org/wiki/Q190411","display_name":"Field-programmable gate array","level":2,"score":0.30059999227523804},{"id":"https://openalex.org/C50630238","wikidata":"https://www.wikidata.org/wiki/Q971505","display_name":"General-purpose computing on graphics processing units","level":3,"score":0.296999990940094},{"id":"https://openalex.org/C2778119891","wikidata":"https://www.wikidata.org/wiki/Q477690","display_name":"CUDA","level":2,"score":0.2930000126361847},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.29249998927116394},{"id":"https://openalex.org/C77390884","wikidata":"https://www.wikidata.org/wiki/Q217302","display_name":"Application-specific integrated circuit","level":2,"score":0.2906000018119812},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.29010000824928284},{"id":"https://openalex.org/C2777575374","wikidata":"https://www.wikidata.org/wiki/Q1644704","display_name":"MicroBlaze","level":3,"score":0.2897000014781952},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.2824999988079071},{"id":"https://openalex.org/C138236772","wikidata":"https://www.wikidata.org/wiki/Q25098575","display_name":"Edge device","level":3,"score":0.26570001244544983},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.257999986410141},{"id":"https://openalex.org/C106516650","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm design","level":2,"score":0.2540999948978424},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.25189998745918274}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3797905.3800530","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3797905.3800530","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th ACM International Conference on Supercomputing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3797905.3800530","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3797905.3800530","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th ACM International Conference on Supercomputing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W2002555321","https://openalex.org/W3208099998","https://openalex.org/W4387321091","https://openalex.org/W4388667495","https://openalex.org/W4389576338","https://openalex.org/W4395106472","https://openalex.org/W4399304401","https://openalex.org/W4406548153","https://openalex.org/W4408843954","https://openalex.org/W4408891420","https://openalex.org/W4412888769","https://openalex.org/W4413360549","https://openalex.org/W4413411760","https://openalex.org/W4413411914","https://openalex.org/W4413411942","https://openalex.org/W4413412033","https://openalex.org/W4415797154","https://openalex.org/W4415797802","https://openalex.org/W7092177792","https://openalex.org/W7133214447","https://openalex.org/W7133224126","https://openalex.org/W7133227460"],"related_works":[],"abstract_inverted_index":{"The":[0,89,234],"high":[1],"computational":[2],"and":[3,114,142,156,170,184,203,230],"memory":[4],"burden":[5],"of":[6,84,91,129,189,228,236],"Large":[7],"Language":[8],"Model":[9],"(LLM)":[10],"inference":[11,78,172],"has":[12],"spurred":[13],"significant":[14,164],"interest":[15],"in":[16],"model":[17],"sparsification.":[18],"While":[19],"unstructured":[20,85],"pruning":[21],"effectively":[22,151,220],"reduces":[23],"parameters":[24],"with":[25,42,117],"minimal":[26,111],"accuracy":[27],"loss,":[28],"achieving":[29],"practical":[30,57,231],"speedups":[31],"on":[32,122,200],"hardware":[33,148],"optimized":[34],"for":[35,87,110],"dense":[36,68,169,194],"matrix":[37],"multiplication,":[38],"such":[39],"as":[40],"GPUs":[41],"Tensor":[43,104],"Cores,":[44],"remains":[45],"a":[46,76,95,127,143],"major":[47],"challenge.":[48],"Traditional":[49],"sparse":[50,63,97,171],"formats":[51],"incur":[52],"substantial":[53],"decoding":[54,112],"overhead":[55,113],"at":[56,215,241],"sparsity":[58,86,217],"levels":[59,218],"(30%\u201390%),":[60],"often":[61],"causing":[62],"computations":[64],"to":[65,146,193,212],"underperform":[66],"their":[67],"counterparts.":[69],"In":[70],"this":[71,123],"paper,":[72],"we":[73],"present":[74],"InferFast,":[75],"high-performance":[77],"framework":[79,198],"that":[80,161],"unlocks":[81],"the":[82,175,222,225],"potential":[83],"LLMs.":[88],"core":[90,119],"our":[92],"approach":[93],"is":[94,238],"novel":[96],"encoding":[98],"format":[99],"called":[100],"Compact":[101],"Dual":[102],"Position":[103],"Core":[105],"Bitmap":[106],"Encoding":[107],"(CDP-TCBE),":[108],"designed":[109],"native":[115],"compatibility":[116],"tensor":[118],"operations.":[120],"Building":[121],"format,":[124],"InferFast":[125,162,207,237],"employs":[126],"suite":[128],"system-level":[130],"optimizations,":[131],"including":[132],"hierarchical":[133],"weight":[134],"reordering,":[135],"efficient":[136],"data":[137,154],"movement,":[138],"vectorized":[139],"bitmap":[140],"decoding,":[141,153],"double-buffered":[144],"pipeline,":[145],"maximize":[147],"utilization":[149],"by":[150,210],"overlapping":[152],"transfer,":[155],"computation.":[157],"Our":[158],"evaluation":[159],"demonstrates":[160],"achieves":[163,185],"performance":[165],"improvements":[166],"over":[167],"state-of-the-art":[168,181],"engines.":[173],"At":[174,196],"kernel":[176],"level,":[177],"it":[178],"significantly":[179],"outperforms":[180],"SpMM":[182],"baselines":[183],"an":[186],"average":[187],"speedup":[188],"2.01":[190],"\u00d7":[191,214],"compared":[192],"GEMM.":[195],"end-to-end":[197],"level":[199],"OPT-30B,":[201],"OPT-66B,":[202],"Llama":[204],"2-13B":[205],"models,":[206],"increases":[208],"throughput":[209],"up":[211],"1.82":[213],"medium":[216],"(70%),":[219],"bridging":[221],"gap":[223],"between":[224],"theoretical":[226],"benefits":[227],"sparsification":[229],"deployment":[232],"efficiency.":[233],"code":[235],"publicly":[239],"available":[240],"https://github.com/MLsys-HPC/InferFast.":[242]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-07-03T00:00:00"}