{"id":"https://openalex.org/W7162648628","doi":"https://doi.org/10.48550/arxiv.2605.28207","title":"Pruning and Distilling Mixture-of-Experts into Dense Language Models","display_name":"Pruning and Distilling Mixture-of-Experts into Dense Language Models","publication_year":2026,"publication_date":"2026-05-27","ids":{"openalex":"https://openalex.org/W7162648628","doi":"https://doi.org/10.48550/arxiv.2605.28207"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.28207","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.28207","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.28207","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137247667","display_name":"Junhyuck Kim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Junhyuck","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110798232","display_name":"Jihun Yun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yun, Jihun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061545561","display_name":"Haechan Kim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Haechan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040969166","display_name":"Gyeongman Kim","orcid":"https://orcid.org/0009-0004-6281-9203"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Gyeongman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113059650","display_name":"Joonghyun Bae","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bae, Joonghyun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137228806","display_name":"Jaewoong Cho","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cho, Jaewoong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.26930001378059387,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.26930001378059387,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1525000035762787,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.08129999786615372,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.796999990940094},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5113999843597412},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.4909000098705292},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.4472000002861023},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.37959998846054077}],"concepts":[{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.796999990940094},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7204999923706055},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5810999870300293},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5175999999046326},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5113999843597412},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.4909000098705292},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.4472000002861023},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.37959998846054077},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3725000023841858},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33640000224113464},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.3319999873638153},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2973000109195709},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2930000126361847}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.28207","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.28207","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.28207","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.28207","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6861624717712402,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Mixture-of-Experts":[0],"(MoE)":[1],"is":[2,118],"now":[3],"the":[4,33,38,45,51,84,113,119],"dominant":[5],"architecture":[6],"for":[7,26,55],"frontier":[8],"language":[9],"models,":[10],"yet":[11],"it":[12,23],"requires":[13],"all":[14],"expert":[15,103],"parameters":[16],"to":[17],"be":[18],"loaded":[19],"in":[20,151],"memory,":[21],"making":[22],"less":[24],"preferable":[25],"memory-constrained":[27],"deployment.":[28],"Existing":[29],"compression":[30],"methods":[31,97,130],"reduce":[32],"number":[34],"of":[35,101,115],"experts":[36,66],"but":[37],"output":[39],"remains":[40],"an":[41],"MoE":[42,59,85],"model":[43],"with":[44,122],"same":[46],"fundamental":[47],"limitation.":[48],"We":[49,87,110],"present":[50],"first":[52],"systematic":[53],"framework":[54],"converting":[56],"a":[57,61,75,99,137],"trained":[58],"into":[60,74],"standard":[62],"fully":[63],"dense":[64,76],"architecture:":[65],"are":[67],"scored,":[68],"selected,":[69],"and":[70,78,93,134],"grouped,":[71],"then":[72],"concatenated":[73],"FFN":[77],"refined":[79],"by":[80,148],"knowledge":[81],"distillation":[82,157],"from":[83],"teacher.":[86],"evaluate":[88],"7":[89],"scoring,":[90],"5":[91],"grouping,":[92],"2":[94],"magnitude":[95],"scaling":[96],"across":[98],"range":[100],"selected":[102],"counts":[104],"on":[105,131],"Qwen3-30B-A3B,":[106,132],"yielding":[107],"350":[108],"configurations.":[109],"find":[111],"that":[112],"choice":[114],"scoring":[116,126],"method":[117],"most":[120],"impactful,":[121],"our":[123],"novel":[124],"diversity-aware":[125],"consistently":[127],"outperforming":[128],"prior":[129],"DeepSeek-V2-Lite,":[133],"GPT-OSS-20B.":[135],"Under":[136],"controlled":[138],"comparison":[139],"at":[140,158],"matched":[141],"parameter":[142],"count,":[143],"MoE-to-dense":[144],"outperforms":[145],"dense-to-dense":[146],"pruning":[147],"+6.3":[149],"pp":[150],"average":[152],"downstream":[153],"accuracy":[154],"after":[155],"~4B-token":[156],"1.6x":[159],"faster":[160],"training":[161],"wall-clock":[162],"speed.":[163]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-29T00:00:00"}