{"id":"https://openalex.org/W4403590326","doi":"https://doi.org/10.48550/arxiv.2409.05929","title":"M3-JEPA: Multimodal Alignment via Multi-gate MoE based on the Joint-Embedding Predictive Architecture","display_name":"M3-JEPA: Multimodal Alignment via Multi-gate MoE based on the Joint-Embedding Predictive Architecture","publication_year":2024,"publication_date":"2024-09-09","ids":{"openalex":"https://openalex.org/W4403590326","doi":"https://doi.org/10.48550/arxiv.2409.05929"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2409.05929","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.05929","pdf_url":"https://arxiv.org/pdf/2409.05929","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2409.05929","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103246867","display_name":"Hongyang Lei","orcid":"https://orcid.org/0009-0001-9705-4595"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lei, Hongyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055078985","display_name":"Xiaolong Cheng","orcid":"https://orcid.org/0009-0009-3271-2364"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Xiaolong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101527063","display_name":"Qi Qin","orcid":"https://orcid.org/0000-0002-5848-4197"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qin, Qi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115595725","display_name":"Dan Wang","orcid":"https://orcid.org/0009-0001-7259-662X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Dan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Fan, Kun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fan, Kun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Huang, Huazhen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Huazhen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066574703","display_name":"Qing Gu","orcid":"https://orcid.org/0000-0002-5948-4690"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gu, Qingqing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047080216","display_name":"Yanchao Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Yetao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104274609","display_name":"Zhonglin Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Zhonglin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100454463","display_name":"Yong Chen","orcid":"https://orcid.org/0000-0003-4054-3395"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5044105175","display_name":"Ligong Ji","orcid":"https://orcid.org/0009-0000-6383-3008"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ji, Luo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9466999769210815,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9466999769210815,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.37351536750793457},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.347136527299881}],"concepts":[{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.37351536750793457},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.347136527299881}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2409.05929","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.05929","pdf_url":"https://arxiv.org/pdf/2409.05929","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2409.05929","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2409.05929","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2409.05929","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.05929","pdf_url":"https://arxiv.org/pdf/2409.05929","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4403590326.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4391375266","https://openalex.org/W1979597421","https://openalex.org/W2007980826","https://openalex.org/W2061531152","https://openalex.org/W3002753104","https://openalex.org/W2077600819","https://openalex.org/W2142036596","https://openalex.org/W2072657027"],"abstract_inverted_index":{"Current":[0],"multimodal":[1,44,119],"learning":[2,165],"strategies":[3],"primarily":[4],"optimize":[5],"in":[6,28,148,166],"the":[7,19,37,43,48,52,62,66,82,91,167],"original":[8],"token":[9],"space.":[10,68],"Such":[11],"a":[12,57,74,160],"framework":[13,83,101],"is":[14,102,145],"easy":[15],"to":[16,139,163],"incorporate":[17],"with":[18,104],"backbone":[20],"of":[21,77],"pretrained":[22],"language":[23],"model,":[24],"but":[25],"might":[26,158],"result":[27],"modality":[29],"collapse.":[30],"To":[31],"alleviate":[32],"such":[33],"issues,":[34],"we":[35,125],"leverage":[36],"Joint-Embedding":[38],"Predictive":[39],"Architecture":[40],"(JEPA)":[41],"on":[42,65,133],"tasks,":[45,137],"which":[46],"converts":[47],"input":[49],"embedding":[50,54],"into":[51],"output":[53],"space":[55],"by":[56,73,112],"predictor":[58,72],"and":[59,80,93,96,107,110,136,142,144,151],"then":[60],"conducts":[61],"cross-modal":[63],"alignment":[64],"latent":[67],"We":[69],"implement":[70],"this":[71],"Multi-Gate":[75],"Mixture":[76],"Experts":[78],"(MMoE)":[79],"name":[81],"as":[84],"M3-JEPA,":[85],"accordingly.":[86],"The":[87,100],"gating":[88],"function":[89],"disentangles":[90],"modality-specific":[92],"shared":[94],"information":[95],"derives":[97],"information-theoretic":[98],"optimality.":[99],"implemented":[103],"both":[105,149],"contrastive":[106],"regularization":[108],"loss,":[109],"solved":[111],"alternative":[113],"gradient":[114],"descent":[115],"(AGD)":[116],"between":[117],"different":[118,134],"tasks.":[120],"By":[121],"thoroughly":[122],"designed":[123],"experiments,":[124],"show":[126],"that":[127,156],"M3-JEPA":[128,157],"can":[129],"obtain":[130],"state-of-the-art":[131],"performance":[132],"modalities":[135],"generalize":[138],"unseen":[140],"datasets":[141],"domains,":[143],"computationally":[146],"efficient":[147],"training":[150],"inference.":[152],"Our":[153],"observation":[154],"suggests":[155],"become":[159],"new":[161],"basis":[162],"self-supervised":[164],"open":[168],"world.":[169]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2024-10-22T00:00:00"}