{"id":"https://openalex.org/W7162419513","doi":"https://doi.org/10.48550/arxiv.2605.25343","title":"Toward Native Multimodal Modeling: A Roadmap","display_name":"Toward Native Multimodal Modeling: A Roadmap","publication_year":2026,"publication_date":"2026-05-25","ids":{"openalex":"https://openalex.org/W7162419513","doi":"https://doi.org/10.48550/arxiv.2605.25343"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.25343","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.25343","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.25343","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137015935","display_name":"Siyu An","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"An, Siyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090635327","display_name":"Junru Lu","orcid":"https://orcid.org/0009-0004-9588-8766"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Junru","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038326486","display_name":"Junnan Dong","orcid":"https://orcid.org/0000-0003-2117-6083"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong, Junnan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136993923","display_name":"Qiufeng Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Qiufeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137000727","display_name":"Yinghui Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yinghui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137030670","display_name":"Weizhi Fei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fei, Weizhi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112840113","display_name":"Zichao Yu","orcid":"https://orcid.org/0009-0005-8880-279X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Zichao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137046579","display_name":"Zheng Yuan","orcid":"https://orcid.org/0009-0003-2543-178X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan, Zheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137032090","display_name":"Biao Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Biao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137073250","display_name":"Haopeng Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Haopeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102592023","display_name":"Renzhao Liang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liang, Renzhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137002221","display_name":"Yixuan Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Yixuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137063440","display_name":"Yunhang Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Yunhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137035941","display_name":"Bo Ke","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ke, Bo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137021106","display_name":"Keyu Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Keyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137048689","display_name":"Linhao Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Linhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137040212","display_name":"Difan Zou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zou, Difan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137065054","display_name":"Xiao Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Xiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100567807","display_name":"Di Yin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Di","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079762822","display_name":"Ruizhi Qiao","orcid":"https://orcid.org/0000-0002-3663-0149"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiao, Ruizhi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137014142","display_name":"Xing Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Xing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.8970999717712402,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.8970999717712402,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.030300000682473183,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.010099999606609344,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.5181999802589417},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.4499000012874603},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.37720000743865967},{"id":"https://openalex.org/keywords/paradigm-shift","display_name":"Paradigm shift","score":0.3698999881744385},{"id":"https://openalex.org/keywords/multimodal-learning","display_name":"Multimodal learning","score":0.350600004196167},{"id":"https://openalex.org/keywords/comprehension","display_name":"Comprehension","score":0.3474999964237213}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6747000217437744},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.5181999802589417},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4499000012874603},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.42260000109672546},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.37720000743865967},{"id":"https://openalex.org/C43540301","wikidata":"https://www.wikidata.org/wiki/Q689971","display_name":"Paradigm shift","level":2,"score":0.3698999881744385},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.350600004196167},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.3474999964237213},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.3425000011920929},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3239000141620636},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.31360000371932983},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.30630001425743103},{"id":"https://openalex.org/C194232998","wikidata":"https://www.wikidata.org/wiki/Q1606712","display_name":"Transition (genetics)","level":3,"score":0.26510000228881836},{"id":"https://openalex.org/C43091099","wikidata":"https://www.wikidata.org/wiki/Q1067788","display_name":"Through-the-lens metering","level":3,"score":0.26330000162124634},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.25290000438690186}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.25343","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.25343","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.25343","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.25343","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Multimodal":[0],"modeling":[1,38,130],"represents":[2],"a":[3,70,136,156],"vital":[4],"step":[5],"from":[6,87,166,169],"modality-agnostic":[7],"reasoning":[8],"toward":[9,35,144],"world":[10],"modeling.":[11,189],"While":[12],"early":[13],"approaches":[14],"predominantly":[15],"rely":[16],"on":[17],"late-fusion":[18],"that":[19],"assembles":[20],"encoders":[21],"and":[22,85,122,125,138,151,182],"frozen":[23],"language":[24],"backbones":[25],"with":[26,40,69,111,131],"output":[27],"heads,":[28],"recent":[29],"efforts":[30],"have":[31],"shifted":[32],"the":[33,41,53,67,80,93,98,142,145,163,183],"paradigm":[34],"native":[36,57,95,188],"multimodal":[37,48],"(NMM)":[39],"intrinsic":[42],"integration":[43],"of":[44,56,100],"modalities":[45],"for":[46,73,108,116,128,186],"superior":[47],"performance.":[49],"Despite":[50],"its":[51],"potential,":[52],"design":[54],"space":[55],"architectures":[58],"remains":[59],"insufficiently":[60],"defined.":[61],"In":[62],"this":[63,74],"paper,":[64],"we":[65,77],"present":[66],"community":[68],"formalized":[71],"roadmap":[72],"transition.":[75],"Specifically,":[76],"formally":[78],"define":[79],"architectural":[81,170],"nativity,":[82],"distinguishing":[83],"mid-fusion":[84],"early-fusion":[86],"non-native":[88],"paradigms.":[89],"We":[90,134,160],"further":[91],"organize":[92],"existing":[94],"models":[96],"through":[97],"lens":[99],"input-output":[101],"duality":[102],"into":[103,141],"three":[104],"categories:":[105],"(i)":[106],"Multi-to-Text":[107],"cross-modal":[109],"comprehension":[110],"text-only":[112],"output;":[113],"(ii)":[114],"Multi-to-Target":[115],"scenario-oriented":[117],"generation,":[118,124],"e.g.,":[119],"image,":[120],"audio":[121],"video":[123],"(iii)":[126],"Multi-to-Multi":[127],"unified":[129,157],"symmetric":[132],"input-output.":[133],"deliver":[135],"comprehensive":[137,184],"industrial-grade":[139],"investigation":[140],"transition":[143],"definitive":[146],"NMM":[147],"framework,":[148],"where":[149],"understanding":[150],"generation":[152],"seamlessly":[153],"coexist":[154],"within":[155],"transformer":[158],"paradigm.":[159],"systematically":[161],"unpack":[162],"end-to-end":[164],"pipeline":[165],"industrial":[167],"perspectives":[168],"coordination,":[171],"massive":[172],"data":[173],"curation,":[174],"to":[175],"full-stack":[176],"training":[177],"recipes,":[178],"inference":[179],"&":[180],"deployment,":[181],"evaluation":[185],"truly":[187]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-27T00:00:00"}