{"id":"https://openalex.org/W6949833054","doi":"https://doi.org/10.5281/zenodo.14213092","title":"MARLIN: Mixed-Precision Auto-Regressive Parallel Inference on Large Language Models","display_name":"MARLIN: Mixed-Precision Auto-Regressive Parallel Inference on Large Language Models","publication_year":2024,"publication_date":"2024-11-24","ids":{"openalex":"https://openalex.org/W6949833054","doi":"https://doi.org/10.5281/zenodo.14213092"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.14213092","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.14213092","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},"type":"software","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.14213092","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Frantar, Elias","orcid":"https://orcid.org/0009-0004-8073-8845"},"institutions":[{"id":"https://openalex.org/I157556583","display_name":"Institute of Science and Technology Austria","ror":"https://ror.org/03gnh5541","country_code":"AT","type":"education","lineage":["https://openalex.org/I157556583"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Frantar, Elias","raw_affiliation_strings":["Institute of Science and Technology Austria"],"raw_orcid":"https://orcid.org/0009-0004-8073-8845","affiliations":[{"raw_affiliation_string":"Institute of Science and Technology Austria","institution_ids":["https://openalex.org/I157556583"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Castro, Roberto L\u00f3pez","orcid":"https://orcid.org/0000-0001-5493-0287"},"institutions":[{"id":"https://openalex.org/I11019714","display_name":"Universidade da Coru\u00f1a","ror":"https://ror.org/01qckj285","country_code":"ES","type":"education","lineage":["https://openalex.org/I11019714"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Castro, Roberto L\u00f3pez","raw_affiliation_strings":["Universidade da Coru\u00f1a"],"raw_orcid":"https://orcid.org/0000-0001-5493-0287","affiliations":[{"raw_affiliation_string":"Universidade da Coru\u00f1a","institution_ids":["https://openalex.org/I11019714"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Chen, Jiale","orcid":"https://orcid.org/0000-0001-5337-5875"},"institutions":[{"id":"https://openalex.org/I157556583","display_name":"Institute of Science and Technology Austria","ror":"https://ror.org/03gnh5541","country_code":"AT","type":"education","lineage":["https://openalex.org/I157556583"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Chen, Jiale","raw_affiliation_strings":["Institute of Science and Technology Austria"],"raw_orcid":"https://orcid.org/0000-0001-5337-5875","affiliations":[{"raw_affiliation_string":"Institute of Science and Technology Austria","institution_ids":["https://openalex.org/I157556583"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Hoefler, Torsten","orcid":"https://orcid.org/0000-0002-1333-9797"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Hoefler, Torsten","raw_affiliation_strings":["ETH Z\u00fcrich"],"raw_orcid":"https://orcid.org/0000-0002-1333-9797","affiliations":[{"raw_affiliation_string":"ETH Z\u00fcrich","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"last","author":{"id":null,"display_name":"Alistarh, Dan","orcid":"https://orcid.org/0000-0003-3650-940X"},"institutions":[{"id":"https://openalex.org/I157556583","display_name":"Institute of Science and Technology Austria","ror":"https://ror.org/03gnh5541","country_code":"AT","type":"education","lineage":["https://openalex.org/I157556583"]},{"id":"https://openalex.org/I4210163042","display_name":"Neural Signals (United States)","ror":"https://ror.org/05cx8ry19","country_code":"US","type":"company","lineage":["https://openalex.org/I4210163042"]}],"countries":["AT","US"],"is_corresponding":false,"raw_author_name":"Alistarh, Dan","raw_affiliation_strings":["Institute of Science and Technology Austria","Neural Magic, Inc"],"raw_orcid":"https://orcid.org/0000-0003-3650-940X","affiliations":[{"raw_affiliation_string":"Institute of Science and Technology Austria","institution_ids":["https://openalex.org/I157556583"]},{"raw_affiliation_string":"Neural Magic, Inc","institution_ids":["https://openalex.org/I4210163042"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.6888999938964844},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6743999719619751},{"id":"https://openalex.org/keywords/ideal","display_name":"Ideal (ethics)","score":0.5303999781608582},{"id":"https://openalex.org/keywords/extension","display_name":"Extension (predicate logic)","score":0.5167999863624573},{"id":"https://openalex.org/keywords/contrast","display_name":"Contrast (vision)","score":0.4332999885082245},{"id":"https://openalex.org/keywords/kernel-method","display_name":"Kernel method","score":0.4153999984264374},{"id":"https://openalex.org/keywords/dual","display_name":"Dual (grammatical number)","score":0.35670000314712524}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7857999801635742},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.6888999938964844},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6743999719619751},{"id":"https://openalex.org/C2776639384","wikidata":"https://www.wikidata.org/wiki/Q840396","display_name":"Ideal (ethics)","level":2,"score":0.5303999781608582},{"id":"https://openalex.org/C2778029271","wikidata":"https://www.wikidata.org/wiki/Q5421931","display_name":"Extension (predicate logic)","level":2,"score":0.5167999863624573},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.4332999885082245},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4203999936580658},{"id":"https://openalex.org/C122280245","wikidata":"https://www.wikidata.org/wiki/Q620622","display_name":"Kernel method","level":3,"score":0.4153999984264374},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3921999931335449},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3711000084877014},{"id":"https://openalex.org/C2780980858","wikidata":"https://www.wikidata.org/wiki/Q110022","display_name":"Dual (grammatical number)","level":2,"score":0.35670000314712524},{"id":"https://openalex.org/C169272836","wikidata":"https://www.wikidata.org/wiki/Q5668307","display_name":"Variable elimination","level":3,"score":0.30880001187324524},{"id":"https://openalex.org/C2777472644","wikidata":"https://www.wikidata.org/wiki/Q16968992","display_name":"Approximate inference","level":3,"score":0.2816999852657318},{"id":"https://openalex.org/C160446489","wikidata":"https://www.wikidata.org/wiki/Q7226642","display_name":"Polynomial kernel","level":4,"score":0.2702000141143799},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.26989999413490295},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2630000114440918},{"id":"https://openalex.org/C179603123","wikidata":"https://www.wikidata.org/wiki/Q1941921","display_name":"Modeling language","level":3,"score":0.25609999895095825},{"id":"https://openalex.org/C137800194","wikidata":"https://www.wikidata.org/wiki/Q11713455","display_name":"Interpolation (computer graphics)","level":3,"score":0.25600001215934753}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.5281/zenodo.14213092","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.14213092","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Software"}],"best_oa_location":{"id":"doi:10.5281/zenodo.14213092","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.14213092","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"This":[0],"is":[1],"Marlin,":[2],"a":[3],"Mixed":[4],"Auto-Regressive":[5],"Linear":[6],"kernel":[7,23],"(and":[8],"the":[9,14,45,61],"name":[10],"of":[11,13,39,48,60],"one":[12],"planet's":[15],"fastest":[16],"fish),":[17],"an":[18,58],"extremely":[19],"optimized":[20],"FP16xINT4":[21],"matmul":[22],"aimed":[24],"at":[25],"LLM":[26],"inference":[27],"that":[28],"can":[29],"deliver":[30],"close":[31],"to":[32,37,44,66],"ideal":[33],"(4x)":[34],"speedups":[35,72],"up":[36],"batchsizes":[38],"16-32":[40],"tokens":[41,47],"(in":[42],"contrast":[43],"1-2":[46],"prior":[49],"work":[50],"with":[51],"comparable":[52],"speedup).":[53],"Additionally,":[54],"it":[55],"includes":[56],"Sparse-Marlin,":[57],"extension":[59],"MARLIN":[62],"kernels":[63],"adding":[64],"support":[65],"2:4":[67],"weight":[68],"sparsity,":[69],"achieving":[70],"5.3x":[71],"on":[73],"NVIDIA":[74],"GPUs":[75],"(Ampere/Ada).":[76]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-10T00:00:00"}