{"id":"https://openalex.org/W4416756395","doi":"https://doi.org/10.1109/dsaa65442.2025.11247985","title":"Small and Fast LLMs on Commodity Hardware: Post-Training Quantization in llama. cpp","display_name":"Small and Fast LLMs on Commodity Hardware: Post-Training Quantization in llama. cpp","publication_year":2025,"publication_date":"2025-10-09","ids":{"openalex":"https://openalex.org/W4416756395","doi":"https://doi.org/10.1109/dsaa65442.2025.11247985"},"language":"en","primary_location":{"id":"doi:10.1109/dsaa65442.2025.11247985","is_oa":false,"landing_page_url":"https://doi.org/10.1109/dsaa65442.2025.11247985","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 12th International Conference on Data Science and Advanced Analytics (DSAA)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086733781","display_name":"Lorenz Sparrenberg","orcid":"https://orcid.org/0000-0001-9450-7387"},"institutions":[{"id":"https://openalex.org/I135140700","display_name":"University of Bonn","ror":"https://ror.org/041nas322","country_code":"DE","type":"education","lineage":["https://openalex.org/I135140700"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Lorenz Sparrenberg","raw_affiliation_strings":["University of Bonn,Bonn,Germany"],"affiliations":[{"raw_affiliation_string":"University of Bonn,Bonn,Germany","institution_ids":["https://openalex.org/I135140700"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003368121","display_name":"Tobias Deu\u00dfer","orcid":"https://orcid.org/0000-0003-4685-0847"},"institutions":[{"id":"https://openalex.org/I135140700","display_name":"University of Bonn","ror":"https://ror.org/041nas322","country_code":"DE","type":"education","lineage":["https://openalex.org/I135140700"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Tobias Deu\u00dfer","raw_affiliation_strings":["University of Bonn,Bonn,Germany"],"affiliations":[{"raw_affiliation_string":"University of Bonn,Bonn,Germany","institution_ids":["https://openalex.org/I135140700"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102998018","display_name":"Armin Berger","orcid":"https://orcid.org/0009-0003-1924-8401"},"institutions":[{"id":"https://openalex.org/I135140700","display_name":"University of Bonn","ror":"https://ror.org/041nas322","country_code":"DE","type":"education","lineage":["https://openalex.org/I135140700"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Armin Berger","raw_affiliation_strings":["University of Bonn,Bonn,Germany"],"affiliations":[{"raw_affiliation_string":"University of Bonn,Bonn,Germany","institution_ids":["https://openalex.org/I135140700"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064201630","display_name":"Rafet Sifa","orcid":"https://orcid.org/0009-0004-6680-8210"},"institutions":[{"id":"https://openalex.org/I135140700","display_name":"University of Bonn","ror":"https://ror.org/041nas322","country_code":"DE","type":"education","lineage":["https://openalex.org/I135140700"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Rafet Sifa","raw_affiliation_strings":["University of Bonn,Bonn,Germany"],"affiliations":[{"raw_affiliation_string":"University of Bonn,Bonn,Germany","institution_ids":["https://openalex.org/I135140700"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5086733781"],"corresponding_institution_ids":["https://openalex.org/I135140700"],"apc_list":null,"apc_paid":null,"fwci":2.3566,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.90945627,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.13230000436306,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.13230000436306,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.12049999833106995,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.08569999784231186,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.7081999778747559},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.6887999773025513},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.4147000014781952},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.3817000091075897},{"id":"https://openalex.org/keywords/de-facto","display_name":"De facto","score":0.2842000126838684}],"concepts":[{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.7081999778747559},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.6887999773025513},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6209999918937683},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4147000014781952},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.3817000091075897},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.2890999913215637},{"id":"https://openalex.org/C2992317946","wikidata":"https://www.wikidata.org/wiki/Q712144","display_name":"De facto","level":2,"score":0.2842000126838684},{"id":"https://openalex.org/C174998907","wikidata":"https://www.wikidata.org/wiki/Q357662","display_name":"Work in process","level":2,"score":0.2808000147342682},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2799000144004822},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27090001106262207},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.26980000734329224},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2653999924659729},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2540999948978424},{"id":"https://openalex.org/C2779439359","wikidata":"https://www.wikidata.org/wiki/Q317088","display_name":"Commodity","level":2,"score":0.2540999948978424}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/dsaa65442.2025.11247985","is_oa":false,"landing_page_url":"https://doi.org/10.1109/dsaa65442.2025.11247985","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 12th International Conference on Data Science and Advanced Analytics (DSAA)","raw_type":"proceedings-article"},{"id":"pmh:oai:bonndoc.ulb.uni-bonn.de:20.500.11811/13751","is_oa":false,"landing_page_url":"https://hdl.handle.net/20.500.11811/13751","pdf_url":null,"source":{"id":"https://openalex.org/S4306402493","display_name":"bonndoc (University of Bonn)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I135140700","host_organization_name":"University of Bonn","host_organization_lineage":["https://openalex.org/I135140700"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"doc-type:conferenceObject"},{"id":"pmh:oai:publica.fraunhofer.de:publica/508165","is_oa":false,"landing_page_url":"https://publica.fraunhofer.de/handle/publica/508165","pdf_url":null,"source":{"id":"https://openalex.org/S4306400318","display_name":"Fraunhofer-Publica (Fraunhofer-Gesellschaft)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4923324","host_organization_name":"Fraunhofer-Gesellschaft","host_organization_lineage":["https://openalex.org/I4923324"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference paper"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":8,"referenced_works":["https://openalex.org/W2963122961","https://openalex.org/W3085139254","https://openalex.org/W4391093139","https://openalex.org/W4402683730","https://openalex.org/W4403693850","https://openalex.org/W4406457849","https://openalex.org/W4406457882","https://openalex.org/W4406495731"],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4],"demonstrated":[5],"remarkable":[6],"capabilities":[7],"but":[8],"their":[9,111],"significant":[10],"computational":[11],"and":[12,42,70,86,106,131,153,168,190],"memory":[13,191],"demands":[14],"hinder":[15],"widespread":[16],"deployment,":[17],"especially":[18],"on":[19,57,124,162],"resource-constrained":[20,163],"devices.":[21],"Quantization,":[22],"the":[23,27,58,65,82,93,119,133,143,170,178,183],"process":[24],"of":[25,30,50,121,135],"reducing":[26],"numerical":[28],"precision":[29],"model":[31,125,186],"parameters,":[32],"has":[33],"emerged":[34],"as":[35,137,149],"a":[36,54,138,150],"critical":[37],"technique":[38],"for":[39,156,197],"compressing":[40],"LLMs":[41,161],"accelerating":[43],"inference.":[44],"This":[45,146,193],"paper":[46,116],"provides":[47],"an":[48],"overview":[49],"LLM":[51],"quantization,":[52],"with":[53,110],"particular":[55],"focus":[56],"Post-Training":[59],"Quantization":[60],"(PTQ)":[61],"methods":[62,172],"implemented":[63],"within":[64,173],"popular":[66],"llama.":[67,99,174],"cpp":[68],"framework":[69],"its":[71],"GGUF":[72,136],"file":[73],"format.":[74],"We":[75,90],"begin":[76],"by":[77,98],"covering":[78],"quantization":[79],"fundamentals,":[80],"including":[81,101],"distinction":[83],"between":[84,185],"PTQ":[85,95,171],"Quantization-Aware":[87],"Training":[88],"(QAT).":[89],"then":[91],"describe":[92],"specific":[94],"schemes":[96],"employed":[97],"cpp,":[100,175],"legacy":[102],"methods,":[103],"advanced":[104],"K-quants,":[105],"recent":[107],"IQ-quants,":[108],"along":[109],"underlying":[112],"mathematical":[113],"principles.":[114],"The":[115],"also":[117],"discusses":[118],"impact":[120],"these":[122],"techniques":[123],"fidelity,":[126,187],"hardware":[127],"requirements,":[128],"inference":[129,188,203],"speed,":[130,189],"traces":[132],"adoption":[134],"de":[139],"facto":[140],"standard":[141],"in":[142],"open-source":[144],"community.":[145],"work":[147],"serves":[148],"practical":[151],"guide":[152],"comprehensive":[154],"reference":[155],"researchers":[157],"aiming":[158],"to":[159,181,204],"deploy":[160],"hardware.":[164],"By":[165],"systematically":[166],"documenting":[167],"comparing":[169],"we":[176],"provide":[177],"necessary":[179],"insights":[180],"navigate":[182],"trade-offs":[184],"footprint.":[192],"enables":[194],"informed":[195],"decision-making":[196],"real-world":[198],"applications,":[199],"from":[200],"local":[201],"CPU-based":[202],"efficient":[205],"edge":[206],"deployment.":[207]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-04-17T18:11:37.981687","created_date":"2025-11-25T00:00:00"}