{"id":"https://openalex.org/W4387964201","doi":"https://doi.org/10.48550/arxiv.2310.16795","title":"QMoE: Practical Sub-1-Bit Compression of Trillion-Parameter Models","display_name":"QMoE: Practical Sub-1-Bit Compression of Trillion-Parameter Models","alternate_titles":[],"publication_year":2023,"publication_date":"2023-10-25","ids":{"openalex":"https://openalex.org/W4387964201","doi":"https://doi.org/10.48550/arxiv.2310.16795"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2310.16795","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.16795","pdf_url":"https://arxiv.org/pdf/2310.16795","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4387964201"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2310.16795","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5060660774","display_name":"Elias Frantar","orcid":"https://orcid.org/0009-0004-8073-8845","observed_orcids":["https://orcid.org/0009-0004-8073-8845"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Frantar, Elias","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5083822059","display_name":"Dan Alistarh","orcid":"https://orcid.org/0000-0003-3650-940X","observed_orcids":["https://orcid.org/0000-0003-3650-940X"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Alistarh, Dan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":63,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9570000171661377,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9570000171661377,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.006800000090152025,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.00559999980032444,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mixture-of-experts","display_name":"mixture of experts","score":0.9520000219345093},{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.7110000252723694},{"id":"https://openalex.org/keywords/sparse-routing","display_name":"sparse routing","score":0.7549999952316284},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9020000100135803},{"id":"https://openalex.org/keywords/memory-efficient-inference","display_name":"memory-efficient inference","score":0.6579999923706055}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8119812607765198},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6755529046058655},{"id":"https://openalex.org/C162478608","wikidata":"https://www.wikidata.org/wiki/Q4011369","display_name":"Uncompressed video","level":4,"score":0.5933915972709656},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.5801936388015747},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5576378107070923},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4572018086910248},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4532701373100281},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.43673816323280334},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.3358907699584961},{"id":"https://openalex.org/C9390403","wikidata":"https://www.wikidata.org/wiki/Q3966","display_name":"Computer hardware","level":1,"score":0.3123665452003479},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3107960820198059},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.18679434061050415},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.15852555632591248},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.08439543843269348},{"id":"https://openalex.org/C202474056","wikidata":"https://www.wikidata.org/wiki/Q1931635","display_name":"Video tracking","level":3,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C65483669","wikidata":"https://www.wikidata.org/wiki/Q3536669","display_name":"Video processing","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2310.16795","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.16795","pdf_url":"https://arxiv.org/pdf/2310.16795","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.16795","doi":"https://doi.org/10.48550/arxiv.2310.16795","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.16795","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2310.16795","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.16795","pdf_url":"https://arxiv.org/pdf/2310.16795","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387964201.pdf","grobid_xml":"https://content.openalex.org/works/W4387964201.grobid-xml"},"referenced_works_count":39,"referenced_works":["https://openalex.org/W1990653637","https://openalex.org/W2060108852","https://openalex.org/W2242818861","https://openalex.org/W2604847669","https://openalex.org/W2900096133","https://openalex.org/W2980282514","https://openalex.org/W2981852735","https://openalex.org/W2986700214","https://openalex.org/W3020212829","https://openalex.org/W3030163527","https://openalex.org/W3040573126","https://openalex.org/W3129093240","https://openalex.org/W3170437634","https://openalex.org/W4221158240","https://openalex.org/W4221165032","https://openalex.org/W4225390052","https://openalex.org/W4226153346","https://openalex.org/W4281776465","https://openalex.org/W4283313765","https://openalex.org/W4287254789","https://openalex.org/W4307934016","https://openalex.org/W4312056202","https://openalex.org/W4313484599","https://openalex.org/W4322718191","https://openalex.org/W4379548477","https://openalex.org/W4384918448","https://openalex.org/W4384919461","https://openalex.org/W4385326807","https://openalex.org/W4385567093","https://openalex.org/W4385571917","https://openalex.org/W4385572286","https://openalex.org/W4385573671","https://openalex.org/W4385968027","https://openalex.org/W4386080707","https://openalex.org/W4408017080","https://openalex.org/W7133198209","https://openalex.org/W7133198810","https://openalex.org/W7133231030","https://openalex.org/W7133233472"],"related_works":["https://openalex.org/W6910774473","https://openalex.org/W4404792840","https://openalex.org/W7104651683","https://openalex.org/W7138400604","https://openalex.org/W7129089911","https://openalex.org/W4403344761","https://openalex.org/W7204168541","https://openalex.org/W4414849752","https://openalex.org/W6929620811","https://openalex.org/W4415331713"],"abstract_inverted_index":{"Mixture-of-Experts":[0],"(MoE)":[1],"architectures":[2],"offer":[3],"a":[4,62,71,83,99,150,153,165,173],"general":[5],"solution":[6,63],"to":[7,47,64,91,108,119,132,192],"the":[8,27,35,126,159,162],"high":[9],"inference":[10],"costs":[11],"of":[12,29,44,70,82,164],"large":[13],"language":[14],"models":[15,201],"(LLMs)":[16],"via":[17],"sparse":[18],"routing,":[19],"bringing":[20],"faster":[21],"and":[22,55,74,199],"more":[23],"accurate":[24],"models,":[25],"at":[26,142,185,204],"cost":[28],"massive":[30],"parameter":[31,129],"counts.":[32],"For":[33],"example,":[34],"SwitchTransformer-c2048":[36,130],"model":[37,131,167],"has":[38],"1.6":[39,127],"trillion":[40,128],"parameters,":[41],"requiring":[42],"3.2TB":[43],"accelerator":[45],"memory":[46,66],"run":[48],"efficiently,":[49],"which":[50,86],"makes":[51],"practical":[52],"deployment":[53],"challenging":[54],"expensive.":[56],"In":[57],"this":[58,65],"paper,":[59],"we":[60],"present":[61],"problem,":[67],"in":[68,98,147],"form":[69],"new":[72],"compression":[73],"execution":[75,163],"framework":[76],"called":[77],"QMoE.":[78],"Specifically,":[79],"QMoE":[80,123],"consists":[81],"scalable":[84],"algorithm":[85],"accurately":[87],"compresses":[88],"trillion-parameter":[89,166],"MoEs":[90],"less":[92,133,148,186],"than":[93,134,149,187],"1":[94],"bit":[95],"per":[96,140],"parameter,":[97],"custom":[100],"format":[101],"co-designed":[102],"with":[103,114,176],"bespoke":[104],"GPU":[105],"decoding":[106],"kernels":[107],"facilitate":[109],"efficient":[110],"end-to-end":[111],"compressed":[112,200],"inference,":[113],"minor":[115,144],"runtime":[116,189],"overheads":[117],"relative":[118,191],"uncompressed":[120,194],"execution.":[121],"Concretely,":[122],"can":[124],"compress":[125],"160GB":[135],"(20x":[136],"compression,":[137],"0.8":[138],"bits":[139],"parameter)":[141],"only":[143],"accuracy":[145],"loss,":[146],"day":[151],"on":[152,168],"single":[154,174],"GPU.":[155],"This":[156],"enables,":[157],"for":[158],"first":[160],"time,":[161],"affordable":[169],"commodity":[170],"hardware,":[171],"like":[172],"server":[175],"4x":[177],"NVIDIA":[178,182],"A6000":[179],"or":[180],"8x":[181],"3090":[183],"GPUs,":[184],"5%":[188],"overhead":[190],"ideal":[193],"inference.":[195],"The":[196],"source":[197],"code":[198],"are":[202],"available":[203],"github.com/IST-DASLab/qmoe.":[205]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4387964201","counts_by_year":[{"year":2026,"cited_by_count":31},{"year":2025,"cited_by_count":20},{"year":2024,"cited_by_count":12}],"updated_date":"2026-11-21T10:27:49.728292","created_date":"2023-10-27T00:00:00"}
