{"id":"https://openalex.org/W4404401018","doi":"https://doi.org/10.1145/3694715.3695964","title":"PowerInfer: Fast Large Language Model Serving with a Consumer-grade GPU","display_name":"PowerInfer: Fast Large Language Model Serving with a Consumer-grade GPU","publication_year":2024,"publication_date":"2024-11-04","ids":{"openalex":"https://openalex.org/W4404401018","doi":"https://doi.org/10.1145/3694715.3695964"},"language":"en","primary_location":{"id":"doi:10.1145/3694715.3695964","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3694715.3695964","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3694715.3695964","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM SIGOPS 30th Symposium on Operating Systems Principles","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3694715.3695964","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081084638","display_name":"Yixin Song","orcid":"https://orcid.org/0009-0001-4605-7382"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yixin Song","raw_affiliation_strings":["Institute of Parallel and Distributed Systems, SEIEE, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0001-4605-7382","affiliations":[{"raw_affiliation_string":"Institute of Parallel and Distributed Systems, SEIEE, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051857167","display_name":"Zeyu Mi","orcid":"https://orcid.org/0000-0001-8395-1319"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zeyu Mi","raw_affiliation_strings":["Institute of Parallel and Distributed Systems, SEIEE, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0001-8395-1319","affiliations":[{"raw_affiliation_string":"Institute of Parallel and Distributed Systems, SEIEE, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102595992","display_name":"Haotong Xie","orcid":null},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haotong Xie","raw_affiliation_strings":["Institute of Parallel and Distributed Systems, SEIEE, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0000-4349-3093","affiliations":[{"raw_affiliation_string":"Institute of Parallel and Distributed Systems, SEIEE, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100406215","display_name":"Haibo Chen","orcid":"https://orcid.org/0000-0002-9720-0361"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haibo Chen","raw_affiliation_strings":["Institute of Parallel and Distributed Systems, SEIEE, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-9720-0361","affiliations":[{"raw_affiliation_string":"Institute of Parallel and Distributed Systems, SEIEE, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I183067930"],"apc_list":null,"apc_paid":null,"fwci":36.9535,"has_fulltext":true,"cited_by_count":113,"citation_normalized_percentile":{"value":0.99881319,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"590","last_page":"606"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.9901000261306763,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9866999983787537,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7991505861282349},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5079501271247864},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics (images)","score":0.3955541253089905}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7991505861282349},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5079501271247864},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.3955541253089905}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3694715.3695964","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3694715.3695964","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3694715.3695964","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM SIGOPS 30th Symposium on Operating Systems Principles","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3694715.3695964","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3694715.3695964","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3694715.3695964","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM SIGOPS 30th Symposium on Operating Systems Principles","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4971838642","display_name":null,"funder_award_id":"61925206","funder_id":"https://openalex.org/F4320334062","funder_display_name":"National Natural Science Foundation of China-Liaoning Joint Fund"},{"id":"https://openalex.org/G863815719","display_name":null,"funder_award_id":"62372287","funder_id":"https://openalex.org/F4320334062","funder_display_name":"National Natural Science Foundation of China-Liaoning Joint Fund"},{"id":"https://openalex.org/G8677931172","display_name":null,"funder_award_id":"61925206","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320334062","display_name":"National Natural Science Foundation of China-Liaoning Joint Fund","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404401018.pdf","grobid_xml":"https://content.openalex.org/works/W4404401018.grobid-xml"},"referenced_works_count":12,"referenced_works":["https://openalex.org/W2023380813","https://openalex.org/W2914304175","https://openalex.org/W2998617917","https://openalex.org/W3130716829","https://openalex.org/W3194676777","https://openalex.org/W4285134706","https://openalex.org/W4321636575","https://openalex.org/W4387302777","https://openalex.org/W4387321091","https://openalex.org/W4388874804","https://openalex.org/W4389576338","https://openalex.org/W4401042327"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"This":[0,48],"paper":[1],"introduces":[2],"PowerInfer,":[3],"a":[4,13,19,42,52,82,155,172,185],"high-speed":[5],"Large":[6],"Language":[7],"Model":[8],"(LLM)":[9],"inference":[10,85],"engine":[11],"on":[12,72,102,154,184],"personal":[14],"computer":[15],"(PC)":[16],"equipped":[17],"with":[18],"single":[20,156,186],"consumer-grade":[21,187],"GPU.":[22,160,190],"The":[23,133],"key":[24],"principle":[25],"underlying":[26],"the":[27,33,66,92,103,125,162],"design":[28,81],"of":[29,55,127,171,179],"PowerInfer":[30,75,115,137,165],"is":[31],"exploiting":[32],"high":[34],"locality":[35],"inherent":[36],"in":[37,45],"LLM":[38],"inference,":[39],"characterized":[40],"by":[41,141],"power-law":[43],"distribution":[44,49],"neuron":[46,128],"activation.":[47],"indicates":[50],"that":[51,136,170],"small":[53],"subset":[54],"neurons,":[56,59,69],"termed":[57],"hot":[58],"are":[60,89,100],"consistently":[61],"activated":[62],"across":[63,149],"inputs,":[64],"while":[65,97,145],"majority,":[67],"cold":[68],"vary":[70],"based":[71],"specific":[73],"inputs.":[74],"exploits":[76],"such":[77],"an":[78],"insight":[79],"to":[80,143,169],"GPU-CPU":[83],"hybrid":[84],"engine:":[86],"hot-activated":[87],"neurons":[88,99],"preloaded":[90],"onto":[91],"GPU":[93,108],"for":[94],"fast":[95],"access,":[96],"cold-activated":[98],"computed":[101],"CPU,":[104],"thus":[105],"significantly":[106,138],"reducing":[107],"memory":[109],"demands":[110],"and":[111,120,130],"CPU-GPU":[112],"data":[113],"transfers.":[114],"further":[116],"integrates":[117],"adaptive":[118],"predictors":[119],"neuron-aware":[121],"sparse":[122],"operators,":[123],"optimizing":[124],"efficiency":[126],"activation":[129],"computational":[131],"sparsity.":[132],"evaluation":[134],"shows":[135],"outperforms":[139],"llama.cpp":[140],"up":[142],"11.69\u00d7":[144],"retaining":[146],"model":[147],"accuracy":[148],"various":[150],"LLMs":[151],"(including":[152],"OPT-175B)":[153],"NVIDIA":[157],"RTX":[158,188],"4090":[159,189],"For":[161],"OPT-30B":[163],"model,":[164],"achieves":[166],"performance":[167],"comparable":[168],"high-end":[173],"server-grade":[174],"A100":[175],"GPU,":[176],"reaching":[177],"82%":[178],"its":[180],"token":[181],"generation":[182],"rate":[183]},"counts_by_year":[{"year":2026,"cited_by_count":50},{"year":2025,"cited_by_count":53},{"year":2024,"cited_by_count":10}],"updated_date":"2026-08-28T12:50:07.497085","created_date":"2025-10-10T00:00:00"}