{"id":"https://openalex.org/W4407451021","doi":"https://doi.org/10.1109/tpami.2025.3532688","title":"Uni-MoE: Scaling Unified Multimodal LLMs With Mixture of Experts","display_name":"Uni-MoE: Scaling Unified Multimodal LLMs With Mixture of Experts","publication_year":2025,"publication_date":"2025-02-13","ids":{"openalex":"https://openalex.org/W4407451021","doi":"https://doi.org/10.1109/tpami.2025.3532688","pmid":"https://pubmed.ncbi.nlm.nih.gov/40031848"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2025.3532688","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3532688","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5019703861","display_name":"Yunxin Li","orcid":"https://orcid.org/0000-0003-4819-2489"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yunxin Li","raw_affiliation_strings":["Department of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100593772","display_name":"Shenyuan Jiang","orcid":null},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shenyuan Jiang","raw_affiliation_strings":["Department of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083079672","display_name":"Baotian Hu","orcid":"https://orcid.org/0000-0001-7490-684X"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Baotian Hu","raw_affiliation_strings":["Department of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088191810","display_name":"Longyue Wang","orcid":"https://orcid.org/0000-0002-9062-6183"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Longyue Wang","raw_affiliation_strings":["Alibaba Group, Hangzhou, China","Meituan and Alibaba Group, China"],"affiliations":[{"raw_affiliation_string":"Alibaba Group, Hangzhou, China","institution_ids":["https://openalex.org/I45928872"]},{"raw_affiliation_string":"Meituan and Alibaba Group, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101215538","display_name":"Wanqi Zhong","orcid":null},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wanqi Zhong","raw_affiliation_strings":["Department of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004450394","display_name":"Wenhan Luo","orcid":"https://orcid.org/0000-0002-5697-4168"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Wenhan Luo","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong"],"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111023136","display_name":"Lin Ma","orcid":null},"institutions":[{"id":"https://openalex.org/I4210087373","display_name":"Meizu (China)","ror":"https://ror.org/0067g4302","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210087373"]},{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lin Ma","raw_affiliation_strings":["Meituan, Beijing, China","Meituan and Alibaba Group, China"],"affiliations":[{"raw_affiliation_string":"Meituan, Beijing, China","institution_ids":["https://openalex.org/I4210087373"]},{"raw_affiliation_string":"Meituan and Alibaba Group, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5108147788","display_name":"Min Zhang","orcid":"https://orcid.org/0000-0003-2864-2307"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Min Zhang","raw_affiliation_strings":["Department of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5019703861"],"corresponding_institution_ids":["https://openalex.org/I204983213"],"apc_list":null,"apc_paid":null,"fwci":67.0917,"has_fulltext":false,"cited_by_count":27,"citation_normalized_percentile":{"value":0.99890305,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"47","issue":"5","first_page":"3424","last_page":"3439"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9825000166893005,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9781000018119812,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5900783538818359},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.5275074243545532},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.482940137386322},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1384250521659851}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5900783538818359},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.5275074243545532},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.482940137386322},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1384250521659851},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/tpami.2025.3532688","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3532688","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:40031848","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40031848","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-146878","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-146878","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":73,"referenced_works":["https://openalex.org/W1494198834","https://openalex.org/W1927052826","https://openalex.org/W2068238590","https://openalex.org/W2560730294","https://openalex.org/W2606964149","https://openalex.org/W2947312908","https://openalex.org/W2963686995","https://openalex.org/W3015591594","https://openalex.org/W3044063956","https://openalex.org/W4206053994","https://openalex.org/W4213019189","https://openalex.org/W4313014461","https://openalex.org/W4321485193","https://openalex.org/W4372340819","https://openalex.org/W4376226279","https://openalex.org/W4382763281","https://openalex.org/W4385245566","https://openalex.org/W4385823465","https://openalex.org/W4386071707","https://openalex.org/W4389519587","https://openalex.org/W4389523832","https://openalex.org/W4390120054","https://openalex.org/W4390874575","https://openalex.org/W4391547487","https://openalex.org/W4392172801","https://openalex.org/W4400033239","https://openalex.org/W4400647053","https://openalex.org/W4402671548","https://openalex.org/W4402672034","https://openalex.org/W4402727764","https://openalex.org/W4403002096","https://openalex.org/W4403793633","https://openalex.org/W6631190155","https://openalex.org/W6635221813","https://openalex.org/W6684090549","https://openalex.org/W6732520560","https://openalex.org/W6771467084","https://openalex.org/W6780805062","https://openalex.org/W6788811087","https://openalex.org/W6791353385","https://openalex.org/W6793102544","https://openalex.org/W6796581206","https://openalex.org/W6805239564","https://openalex.org/W6838434436","https://openalex.org/W6847076894","https://openalex.org/W6847363464","https://openalex.org/W6848151458","https://openalex.org/W6848208918","https://openalex.org/W6849177959","https://openalex.org/W6850071225","https://openalex.org/W6851592950","https://openalex.org/W6851950068","https://openalex.org/W6852489829","https://openalex.org/W6852776751","https://openalex.org/W6853094705","https://openalex.org/W6853116092","https://openalex.org/W6853838016","https://openalex.org/W6854262950","https://openalex.org/W6854510102","https://openalex.org/W6854866820","https://openalex.org/W6855055401","https://openalex.org/W6855297460","https://openalex.org/W6856794988","https://openalex.org/W6856800273","https://openalex.org/W6856974625","https://openalex.org/W6857241981","https://openalex.org/W6858379761","https://openalex.org/W6859304583","https://openalex.org/W6859532197","https://openalex.org/W6860041859","https://openalex.org/W6860710830","https://openalex.org/W6861475616","https://openalex.org/W6917636526"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Recent":[0],"advancements":[1],"in":[2,193],"Multimodal":[3],"Large":[4],"Language":[5],"Models":[6],"(MLLMs)":[7],"underscore":[8],"the":[9,27,59,68,101,119,155,170],"significance":[10],"of":[11,29,79,177,188],"scalable":[12],"models":[13,42],"and":[14,50,107,113,122,152,202],"data":[15,111,147],"to":[16,36,62,103,148],"boost":[17],"performance,":[18],"yet":[19],"this":[20],"often":[21],"incurs":[22],"substantial":[23],"computational":[24],"costs.":[25],"Although":[26],"Mixture":[28],"Experts":[30],"(MoE)":[31],"architecture":[32,99],"has":[33],"been":[34],"employed":[35],"scale":[37],"large":[38],"language":[39],"or":[40],"visual-language":[41],"efficiently,":[43],"these":[44],"efforts":[45],"typically":[46],"involve":[47],"fewer":[48],"experts":[49,143],"limited":[51],"modalities.":[52,80],"To":[53,117],"address":[54],"this,":[55],"our":[56],"work":[57],"presents":[58],"pioneering":[60],"attempt":[61],"develop":[63],"a":[64,76,89,96,126,174],"unified":[65,90],"MLLM":[66],"with":[67,86,136,144],"MoE":[69,98],"architecture,":[70],"named":[71],"Uni-MoE":[72,157,172],"that":[73],"can":[74],"handle":[75],"wide":[77],"array":[78],"Specifically,":[81],"it":[82],"features":[83],"modality-specific":[84,142],"encoders":[85],"connectors":[87,135],"for":[88],"multimodal":[91,165,178,196],"representation.":[92],"We":[93,168],"also":[94],"implement":[95],"sparse":[97],"within":[100],"LLMs":[102],"enable":[104],"efficient":[105],"training":[106,128],"inference":[108],"through":[109],"modality-level":[110],"parallelism":[112],"expert-level":[114],"model":[115],"parallelism.":[116],"enhance":[118],"multi-expert":[120,200],"collaboration":[121,201],"generalization,":[123],"we":[124],"present":[125],"progressive":[127],"strategy:":[129],"1)":[130],"Cross-modality":[131],"alignment":[132],"using":[133],"various":[134],"different":[137],"cross-modality":[138,145],"data,":[139],"2)":[140],"Training":[141],"instruction":[146,166],"activate":[149],"experts'":[150],"preferences,":[151],"3)":[153],"Tuning":[154],"whole":[156],"framework":[158],"utilizing":[159],"Low-Rank":[160],"Adaptation":[161],"(LoRA)":[162],"on":[163,173],"mixed":[164,195],"data.":[167],"evaluate":[169],"instruction-tuned":[171],"comprehensive":[175],"set":[176],"datasets.":[179],"The":[180],"extensive":[181],"experimental":[182],"results":[183],"demonstrate":[184],"Uni-MoE's":[185],"principal":[186],"advantage":[187],"significantly":[189],"reducing":[190],"performance":[191],"bias":[192],"handling":[194],"datasets,":[197],"alongside":[198],"improved":[199],"generalization.":[203]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":19}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}