{"id":"https://openalex.org/W4412945078","doi":"https://doi.org/10.18653/v1/2025.acl-long.816","title":"Automatic Expert Discovery in LLM Upcycling via Sparse Interpolated Mixture-of-Experts","display_name":"Automatic Expert Discovery in LLM Upcycling via Sparse Interpolated Mixture-of-Experts","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412945078","doi":"https://doi.org/10.18653/v1/2025.acl-long.816"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.acl-long.816","doi":"https://doi.org/10.18653/v1/2025.acl-long.816","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.816","pdf_url":"https://aclanthology.org/2025.acl-long.816.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2025.acl-long.816.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058266998","display_name":"Shengzhuang Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210111299","display_name":"Thomson Reuters Foundation","ror":"https://ror.org/01xpmbc27","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210111299"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shengzhuang Chen","raw_affiliation_strings":["Thomson Reuters Foundational Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Thomson Reuters Foundational Research","institution_ids":["https://openalex.org/I4210111299"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011883201","display_name":"Ying Wei","orcid":"https://orcid.org/0000-0003-1662-4443","observed_orcids":["https://orcid.org/0000-0003-1662-4443"]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ying Wei","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5012775658","display_name":"Jonathan Richard Schwarz","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210111299","display_name":"Thomson Reuters Foundation","ror":"https://ror.org/01xpmbc27","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210111299"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jonathan Richard Schwarz","raw_affiliation_strings":["Thomson Reuters Foundational Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Thomson Reuters Foundational Research","institution_ids":["https://openalex.org/I4210111299"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":93},"biblio":{"volume":null,"issue":null,"first_page":"16703","last_page":"16717"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.5054000020027161,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.5054000020027161,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4187000095844269,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.05480000004172325,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mixture-of-experts","display_name":"mixture of experts","score":0.8460000157356262},{"id":"https://openalex.org/keywords/sparse-interpolation","display_name":"sparse interpolation","score":0.656000018119812},{"id":"https://openalex.org/keywords/instruction-tuning","display_name":"instruction tuning","score":0.671999990940094},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8240000009536743},{"id":"https://openalex.org/keywords/router-networks","display_name":"router networks","score":0.6909999847412109},{"id":"https://openalex.org/keywords/parameter-sparsity","display_name":"parameter sparsity","score":0.7310000061988831}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6495803594589233},{"id":"https://openalex.org/C74187038","wikidata":"https://www.wikidata.org/wiki/Q1418791","display_name":"Drug discovery","level":2,"score":0.5197910070419312},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39710018038749695},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32173359394073486},{"id":"https://openalex.org/C60644358","wikidata":"https://www.wikidata.org/wiki/Q128570","display_name":"Bioinformatics","level":1,"score":0.11468160152435303},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2025.acl-long.816","doi":"https://doi.org/10.18653/v1/2025.acl-long.816","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.816","pdf_url":"https://aclanthology.org/2025.acl-long.816.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2506.12597","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.12597","pdf_url":"https://arxiv.org/pdf/2506.12597","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2506.12597","doi":"https://doi.org/10.48550/arxiv.2506.12597","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2506.12597","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.acl-long.816","doi":"https://doi.org/10.18653/v1/2025.acl-long.816","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.816","pdf_url":"https://aclanthology.org/2025.acl-long.816.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412945078.pdf","grobid_xml":"https://content.openalex.org/works/W4412945078.grobid-xml"},"referenced_works_count":9,"referenced_works":["https://openalex.org/W4300886482","https://openalex.org/W4327810158","https://openalex.org/W4390723197","https://openalex.org/W4392823160","https://openalex.org/W4394593019","https://openalex.org/W4401306886","https://openalex.org/W4402671950","https://openalex.org/W4403364470","https://openalex.org/W4404987838"],"related_works":["https://openalex.org/W4378473852","https://openalex.org/W4412945365","https://openalex.org/W4404782838","https://openalex.org/W7166805358","https://openalex.org/W4404783368","https://openalex.org/W4415036992","https://openalex.org/W7166803187","https://openalex.org/W4402670745","https://openalex.org/W4403585466","https://openalex.org/W4400267199"],"abstract_inverted_index":{"We":[0],"present":[1],"Sparse":[2],"Interpolated":[3],"Mixture-of-Experts":[4],"(SIMoE)":[5],"instruction-tuning,":[6,31],"an":[7,66,98],"endto-end":[8],"algorithm":[9],"designed":[10],"to":[11,58,103],"fine-tune":[12],"a":[13,21,39,47,72],"dense":[14],"pre-trained":[15],"Large":[16],"Language":[17],"Model":[18],"(LLM)":[19],"into":[20],"MoE-style":[22],"model":[23],"that":[24,56,83],"possesses":[25],"capabilities":[26],"in":[27],"multiple":[28,35],"specialized":[29,36],"domains.During":[30],"SIMoE":[32,87],"automatically":[33],"identifies":[34],"experts":[37],"under":[38],"specified":[40],"sparsity":[41],"constraint,":[42],"with":[43],"each":[44],"expert":[45,68],"representing":[46],"structurally":[48],"sparse":[49],"subset":[50],"of":[51],"the":[52,62],"seed":[53],"LLM's":[54],"parameters":[55],"correspond":[57],"domainspecific":[59],"knowledge":[60,78],"within":[61],"data.SIMoE":[63],"simultaneously":[64],"learns":[65],"input-dependent":[67],"merging":[69],"strategy":[70],"via":[71],"router":[73],"network,":[74],"leveraging":[75],"rich":[76],"cross-expert":[77],"for":[79],"superior":[80],"downstream":[81],"generalization":[82],"surpasses":[84],"existing":[85],"baselines.Empirically,":[86],"consistently":[88],"achieves":[89],"state-of-the-art":[90],"performance":[91],"on":[92],"common":[93],"instruction-tuning":[94],"benchmarks":[95],"while":[96],"maintaining":[97],"optimal":[99],"performance-compute":[100],"trade-off":[101],"compared":[102],"all":[104],"baselines.":[105]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-10T00:00:00"}
