{"id":"https://openalex.org/W4416510431","doi":"https://doi.org/10.18653/v1/2026.acl-long.34","title":"MoCa: Modality-aware Continual Pre-training Makes Better Bidirectional Multimodal Embeddings","display_name":"MoCa: Modality-aware Continual Pre-training Makes Better Bidirectional Multimodal Embeddings","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W4416510431","doi":"https://doi.org/10.18653/v1/2026.acl-long.34"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2026.acl-long.34","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.34","pdf_url":"https://aclanthology.org/2026.acl-long.34.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.acl-long.34.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100767772","display_name":"Haonan Chen","orcid":"https://orcid.org/0009-0008-3015-0575"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haonan Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032709433","display_name":"Hongyuan Liu","orcid":"https://orcid.org/0000-0002-5437-4695"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hong Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102363880","display_name":"Yuping Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuping Luo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041748805","display_name":"Liang Wang","orcid":"https://orcid.org/0000-0002-0136-6747"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liang Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100680993","display_name":"Nan Yang","orcid":"https://orcid.org/0000-0002-9373-5289"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nan Yang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014662947","display_name":"Furu Wei","orcid":"https://orcid.org/0000-0002-7810-5852"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Furu Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5010558184","display_name":"Zhicheng Dou","orcid":"https://orcid.org/0000-0002-9781-948X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhicheng Dou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.02210625,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"810","last_page":"823"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9495000243186951,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9495000243186951,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.014399999752640724,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.010900000110268593,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.7731000185012817},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7235999703407288},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.5627999901771545},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.5566999912261963},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5268999934196472},{"id":"https://openalex.org/keywords/face","display_name":"Face (sociological concept)","score":0.4690000116825104},{"id":"https://openalex.org/keywords/joint","display_name":"Joint (building)","score":0.4643000066280365},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.4083999991416931}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7939000129699707},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.7731000185012817},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7235999703407288},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5824000239372253},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5627999901771545},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.5566999912261963},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5268999934196472},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.4690000116825104},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.4643000066280365},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43389999866485596},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4083999991416931},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.384799987077713},{"id":"https://openalex.org/C112313634","wikidata":"https://www.wikidata.org/wiki/Q7886648","display_name":"Complement (music)","level":5,"score":0.3513999879360199},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.3041999936103821},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.3027999997138977},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.29190000891685486},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.2912999987602234},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2856999933719635},{"id":"https://openalex.org/C116409475","wikidata":"https://www.wikidata.org/wiki/Q1385056","display_name":"External Data Representation","level":2,"score":0.28209999203681946},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.2815999984741211},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2775999903678894},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.27639999985694885},{"id":"https://openalex.org/C31170391","wikidata":"https://www.wikidata.org/wiki/Q188619","display_name":"Hierarchy","level":2,"score":0.2630000114440918},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.26190000772476196},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.257999986410141}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2026.acl-long.34","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.34","pdf_url":"https://aclanthology.org/2026.acl-long.34.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2506.23115","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.23115","pdf_url":"https://arxiv.org/pdf/2506.23115","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2506.23115","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2506.23115","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.acl-long.34","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.34","pdf_url":"https://aclanthology.org/2026.acl-long.34.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G228125052","display_name":null,"funder_award_id":"62272467","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322724","display_name":"Ministry of Education, India","ror":"https://ror.org/048xjjh50"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416510431.pdf","grobid_xml":"https://content.openalex.org/works/W4416510431.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Multimodal":[0],"embedding":[1,29,68],"models,":[2],"built":[3],"upon":[4],"causal":[5,21],"Vision":[6],"Language":[7],"Models":[8],"(VLMs),":[9],"have":[10,169],"shown":[11],"promise":[12],"in":[13,23,47],"various":[14],"tasks.However,":[15],"current":[16],"approaches":[17],"face":[18],"three":[19],"limitations:":[20],"attention":[22,119],"VLM":[24],"backbones":[25],"is":[26],"suboptimal":[27],"for":[28,41,61,138],"tasks;":[30],"scalability":[31,159],"issues":[32],"due":[33],"to":[34,106],"reliance":[35],"on":[36,167,176],"high-quality":[37],"labeled":[38],"paired":[39],"data":[40,101,137,166,175],"contrastive":[42],"learning;":[43],"and":[44,50,85,109,133,150,156,164,174],"limited":[45],"diversity":[46],"training":[48,165],"objectives":[49],"data.To":[51],"address":[52],"these":[53],"issues,":[54],"we":[55],"propose":[56],"MoCa,":[57],"a":[58,76],"two-stage":[59],"framework":[60],"transforming":[62],"pre-trained":[63],"VLMs":[64],"into":[65],"bidirectional":[66,88,118],"multimodal":[67,100,136],"models.The":[69],"first":[70],"stage,":[71,92],"Modality-aware":[72],"Continual":[73],"Pre-training,":[74],"introduces":[75],"joint":[77,130],"reconstruction":[78,131],"objective":[79],"that":[80,143],"simultaneously":[81],"denoises":[82],"interleaved":[83],"texts":[84],"images,":[86],"enhancing":[87],"context-aware":[89],"reasoning.The":[90],"second":[91],"Heterogeneous":[93],"Contrastive":[94],"Fine-tuning,":[95],"leverages":[96],"diverse,":[97],"semantically":[98],"rich":[99],"beyond":[102],"simple":[103],"image-caption":[104],"pairs":[105],"enhance":[107],"generalization":[108],"alignment.Our":[110],"method":[111],"addresses":[112],"the":[113,171],"stated":[114],"limitations":[115],"by":[116],"introducing":[117],"through":[120],"continual":[121],"pre-training,":[122],"scaling":[123],"effectively":[124],"with":[125,160],"massive":[126],"unlabeled":[127],"datasets":[128],"via":[129],"objectives,":[132],"utilizing":[134],"diverse":[135],"enhanced":[139],"representation":[140],"robustness.Experiments":[141],"demonstrate":[142],"MoCa":[144],"consistently":[145],"improves":[146],"performance":[147],"across":[148],"MMEB":[149],"ViDoRe-v2":[151],"benchmarks,":[152],"achieving":[153],"new":[154],"state-of-the-arts,":[155],"exhibits":[157],"strong":[158],"both":[161],"model":[162,172],"size":[163],"MMEB.We":[168],"released":[170],"weights":[173],"our":[177],"project":[178],"page":[179]},"counts_by_year":[],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2025-10-10T00:00:00"}