{"id":"https://openalex.org/W4412945365","doi":"https://doi.org/10.18653/v1/2025.acl-long.637","title":"Upcycling Instruction Tuning from Dense to Mixture-of-Experts via Parameter Merging","display_name":"Upcycling Instruction Tuning from Dense to Mixture-of-Experts via Parameter Merging","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412945365","doi":"https://doi.org/10.18653/v1/2025.acl-long.637"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.acl-long.637","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.637","pdf_url":"https://aclanthology.org/2025.acl-long.637.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.acl-long.637.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113204589","display_name":"Tingfeng Hui","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tingfeng Hui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119204500","display_name":"Zhenyu Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhenyu Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101351951","display_name":"Shuohuan Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shuohuan Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100701265","display_name":"Yu Sun","orcid":"https://orcid.org/0000-0002-5029-0224"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu Sun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071323362","display_name":"Hua Wu","orcid":"https://orcid.org/0000-0002-9883-4091"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hua Wu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5036865453","display_name":"Sen Su","orcid":"https://orcid.org/0000-0003-4266-7527"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sen Su","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.08883186,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"13012","last_page":"13031"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9419000148773193,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9419000148773193,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6075278520584106}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6075278520584106}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.acl-long.637","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.637","pdf_url":"https://aclanthology.org/2025.acl-long.637.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.acl-long.637","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.637","pdf_url":"https://aclanthology.org/2025.acl-long.637.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1755690126","display_name":"\u9690\u79c1\u4fdd\u62a4\u7684\u6570\u636e\u96c6\u6210\u4e0e\u5171\u4eab\u5173\u952e\u6280\u672f\u7814\u7a76","funder_award_id":"62072052","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412945365.pdf","grobid_xml":"https://content.openalex.org/works/W4412945365.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Mixture-of-Experts":[0],"(MoE)":[1],"shines":[2],"brightly":[3],"in":[4,13,116,161,174],"large":[5],"language":[6,16],"models":[7,89],"(LLMs)":[8],"and":[9,31,79,98,144,151],"demonstrates":[10],"outstanding":[11,149],"performance":[12,150],"plentiful":[14],"natural":[15],"processing":[17],"tasks.However,":[18],"existing":[19],"methods":[20],"transforming":[21],"LLMs":[22],"from":[23],"dense":[24,51,71],"to":[25,86,103,135],"MoE":[26,56,118],"face":[27],"significant":[28],"data":[29,130,142,152,164],"requirements":[30],"typically":[32],"rely":[33],"on":[34],"large-scale":[35],"post-training.In":[36],"this":[37],"paper,":[38],"we":[39,59,123],"propose":[40,81],"Upcycling":[41],"Instruction":[42],"Tuning":[43],"(UpIT),":[44],"a":[45,50,55,125],"data-efficient":[46],"approach":[47],"for":[48,76],"tuning":[49,68],"pre-trained":[52],"model":[53,72,119],"into":[54],"instruction":[57,67],"model.Specifically,":[58],"first":[60],"point":[61],"out":[62],"that":[63,112,131],"intermediate":[64],"checkpoints":[65],"during":[66],"of":[69,93,107,128,154,170],"the":[70,117,137,148,168],"are":[73,101],"naturally":[74],"suitable":[75],"specialized":[77,114],"experts,":[78,94],"then":[80],"an":[82],"expert":[83,115,133,162,172],"expansion":[84],"stage":[85],"flexibly":[87],"achieve":[88],"with":[90,140],"flexible":[91],"numbers":[92],"where":[95],"genetic":[96],"algorithm":[97],"parameter":[99],"merging":[100],"introduced":[102],"ensure":[104,111],"sufficient":[105],"diversity":[106,173],"new":[108],"extended":[109],"experts.To":[110],"each":[113,132],"works":[120],"as":[121,156,158],"expected,":[122],"select":[124],"small":[126],"amount":[127],"seed":[129],"excels":[134],"preoptimize":[136],"router.Extensive":[138],"experiments":[139],"various":[141],"scales":[143],"upcycling":[145],"settings":[146],"demonstrate":[147],"efficiency":[153],"UpIT,":[155],"well":[157],"stable":[159],"improvement":[160],"or":[163],"scaling.Further":[165],"analysis":[166],"reveals":[167],"importance":[169],"ensuring":[171],"upcycling.":[175]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}