{"id":"https://openalex.org/W4405033040","doi":"https://doi.org/10.48550/arxiv.2411.19146","title":"Puzzle: Distillation-Based NAS for Inference-Optimized LLMs","display_name":"Puzzle: Distillation-Based NAS for Inference-Optimized LLMs","publication_year":2024,"publication_date":"2024-11-28","ids":{"openalex":"https://openalex.org/W4405033040","doi":"https://doi.org/10.48550/arxiv.2411.19146"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2411.19146","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.19146","pdf_url":"https://arxiv.org/pdf/2411.19146","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2411.19146","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5115003438","display_name":"Akhiad Bercovich","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bercovich, Akhiad","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003445","display_name":"Tomer Ronen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ronen, Tomer","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003452","display_name":"Talor Abramovich","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Abramovich, Talor","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033770678","display_name":"Nir Ailon","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ailon, Nir","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003453","display_name":"Nave Assaf","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Assaf, Nave","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003454","display_name":"Mohammad Dabbah","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dabbah, Mohammad","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031244684","display_name":"Ido Galil","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Galil, Ido","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003455","display_name":"Amnon Geifman","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Geifman, Amnon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061409282","display_name":"Yonatan Geifman","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Geifman, Yonatan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012929891","display_name":"Izhak Golan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Golan, Izhak","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003439","display_name":"Netanel Haber","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haber, Netanel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003440","display_name":"Ehud Karpas","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karpas, Ehud","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Koren, Roi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Koren, Roi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073884595","display_name":"Itay Levy","orcid":"https://orcid.org/0000-0002-1689-4032"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Levy, Itay","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066945976","display_name":"Pavlo Molchanov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Molchanov, Pavlo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003442","display_name":"Shahar Mor","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mor, Shahar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056351863","display_name":"Zach Moshe","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Moshe, Zach","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003443","display_name":"Najeeb Nabwani","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nabwani, Najeeb","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067683763","display_name":"Omri Puny","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Puny, Omri","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003446","display_name":"Ran Rubin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rubin, Ran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003447","display_name":"Itamar Schen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Schen, Itamar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003448","display_name":"Ido Shahaf","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shahaf, Ido","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003449","display_name":"Oren Tropp","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tropp, Oren","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003450","display_name":"Omer Ullman Argov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Argov, Omer Ullman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115003451","display_name":"Ran Zilberstein","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zilberstein, Ran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5025669300","display_name":"Ran El\u2010Yaniv","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"El-Yaniv, Ran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.8838000297546387,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.8838000297546387,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6408872008323669},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.4942188560962677},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.33908796310424805},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.33840757608413696},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.19992464780807495},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.18800723552703857},{"id":"https://openalex.org/keywords/chromatography","display_name":"Chromatography","score":0.11296960711479187}],"concepts":[{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6408872008323669},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.4942188560962677},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.33908796310424805},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.33840757608413696},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.19992464780807495},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.18800723552703857},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.11296960711479187}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2411.19146","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.19146","pdf_url":"https://arxiv.org/pdf/2411.19146","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2411.19146","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2411.19146","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2411.19146","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.19146","pdf_url":"https://arxiv.org/pdf/2411.19146","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4405033040.pdf","grobid_xml":"https://content.openalex.org/works/W4405033040.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4391375266","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,58,98,103,131,166,183],"(LLMs)":[3],"offer":[4],"remarkable":[5],"capabilities,":[6],"yet":[7],"their":[8,46],"high":[9],"inference":[10,41,107,136,198],"costs":[11],"restrict":[12],"wider":[13],"adoption.":[14],"While":[15],"increasing":[16],"parameter":[17,201],"counts":[18],"improves":[19],"accuracy,":[20],"it":[21],"also":[22],"broadens":[23],"the":[24,40,121,128,151,171],"gap":[25],"between":[26],"state-of-the-art":[27],"capabilities":[28],"and":[29,77,93],"practical":[30],"deployability.":[31],"We":[32,85],"present":[33],"Puzzle,":[34],"a":[35,54,105,112],"hardware-aware":[36],"framework":[37],"that":[38,160,180,197],"accelerates":[39],"of":[42,61,63,120],"LLMs":[43],"while":[44,117],"preserving":[45],"capabilities.":[47,176],"Using":[48],"neural":[49],"architecture":[50,75],"search":[51],"(NAS)":[52],"at":[53,146],"large-scale,":[55],"Puzzle":[56],"optimizes":[57],"with":[59,137,190],"tens":[60],"billions":[62],"parameters.":[64],"Our":[65,177],"approach":[66],"utilizes":[67],"blockwise":[68],"local":[69],"knowledge":[70],"distillation":[71],"(BLD)":[72],"for":[73,81,187],"parallel":[74],"exploration":[76],"employs":[78],"mixed-integer":[79],"programming":[80],"precise":[82],"constraint":[83],"optimization.":[84],"showcase":[86],"our":[87],"framework's":[88],"impact":[89],"via":[90],"Llama-3.1-Nemotron-51B-Instruct":[91],"(Nemotron-51B)":[92],"Llama-3.3-Nemotron-49B,":[94],"two":[95],"publicly":[96],"available":[97],"derived":[99,165],"from":[100],"Llama-70B-Instruct.":[101],"Both":[102],"achieve":[104],"2.17x":[106],"throughput":[108],"speedup,":[109],"fitting":[110],"on":[111,143,163],"single":[113,133],"NVIDIA":[114],"H100":[115,134],"GPU":[116,135],"retaining":[118],"98.4%":[119],"original":[122],"model's":[123],"benchmark":[124],"accuracies.":[125],"These":[126],"are":[127],"most":[129],"accurate":[130],"supporting":[132],"large":[138],"batch":[139],"sizes,":[140],"despite":[141],"training":[142],"45B":[144],"tokens":[145],"most,":[147],"far":[148],"fewer":[149],"than":[150],"15T":[152],"used":[153],"to":[154,169],"train":[155],"Llama-70B.":[156],"Lastly,":[157],"we":[158],"show":[159],"lightweight":[161],"alignment":[162],"these":[164],"allows":[167],"them":[168],"surpass":[170],"parent":[172],"model":[173,206],"in":[174,194],"specific":[175],"work":[178],"establishes":[179],"powerful":[181],"LLM":[182],"can":[184],"be":[185],"optimized":[186],"efficient":[188],"deployment":[189],"only":[191],"negligible":[192],"loss":[193],"quality,":[195],"underscoring":[196],"performance,":[199],"not":[200],"count":[202],"alone,":[203],"should":[204],"guide":[205],"selection.":[207]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-10-10T00:00:00"}