{"id":"https://openalex.org/W4394606408","doi":"https://doi.org/10.1145/3656580","title":"Foundations & Trends in Multimodal Machine Learning: Principles, Challenges, and Open Questions","display_name":"Foundations & Trends in Multimodal Machine Learning: Principles, Challenges, and Open Questions","publication_year":2024,"publication_date":"2024-04-09","ids":{"openalex":"https://openalex.org/W4394606408","doi":"https://doi.org/10.1145/3656580"},"language":"en","primary_location":{"id":"doi:10.1145/3656580","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3656580","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3656580","source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"type":"review","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3656580","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086233510","display_name":"Paul Pu Liang","orcid":"https://orcid.org/0000-0001-7768-3610"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Paul Pu Liang","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112033266","display_name":"Amir Zadeh","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amir Zadeh","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081398601","display_name":"Louis\u2013Philippe Morency","orcid":"https://orcid.org/0000-0001-6376-7696"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Louis-Philippe Morency","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, United States"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, United States","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5086233510"],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":32.8108,"has_fulltext":true,"cited_by_count":136,"citation_normalized_percentile":{"value":0.99874289,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"56","issue":"10","first_page":"1","last_page":"42"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8417247533798218},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4421222507953644},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4038122892379761},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.40286785364151},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3200693130493164}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8417247533798218},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4421222507953644},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4038122892379761},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.40286785364151},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3200693130493164}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3656580","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3656580","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3656580","source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3656580","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3656580","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3656580","source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.49000000953674316,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4394606408.pdf","grobid_xml":"https://content.openalex.org/works/W4394606408.grobid-xml"},"referenced_works_count":300,"referenced_works":["https://openalex.org/W102708294","https://openalex.org/W1520777553","https://openalex.org/W1529533208","https://openalex.org/W1566289585","https://openalex.org/W1582211066","https://openalex.org/W1773149199","https://openalex.org/W1882423120","https://openalex.org/W1906515132","https://openalex.org/W1911418635","https://openalex.org/W1965807894","https://openalex.org/W1967259204","https://openalex.org/W1981613567","https://openalex.org/W1981974552","https://openalex.org/W1989689139","https://openalex.org/W1995875735","https://openalex.org/W2005505771","https://openalex.org/W2011699475","https://openalex.org/W2013489329","https://openalex.org/W2022166150","https://openalex.org/W2024082504","https://openalex.org/W2026012689","https://openalex.org/W2026916126","https://openalex.org/W2029996593","https://openalex.org/W2035299679","https://openalex.org/W2038227658","https://openalex.org/W2041649634","https://openalex.org/W2042608483","https://openalex.org/W2048231652","https://openalex.org/W2048679005","https://openalex.org/W2053101950","https://openalex.org/W2057568625","https://openalex.org/W2058787788","https://openalex.org/W2081580037","https://openalex.org/W2082453965","https://openalex.org/W2085789144","https://openalex.org/W2089150756","https://openalex.org/W2094728533","https://openalex.org/W2102468971","https://openalex.org/W2102631167","https://openalex.org/W2104424090","https://openalex.org/W2106277773","https://openalex.org/W2108036388","https://openalex.org/W2112912048","https://openalex.org/W2116341502","https://openalex.org/W2128614648","https://openalex.org/W2130055251","https://openalex.org/W2131953535","https://openalex.org/W2143891888","https://openalex.org/W2144345993","https://openalex.org/W2155279768","https://openalex.org/W2159527087","https://openalex.org/W2163922914","https://openalex.org/W2164378405","https://openalex.org/W2168180613","https://openalex.org/W2170326347","https://openalex.org/W2171939880","https://openalex.org/W2185175083","https://openalex.org/W2218566711","https://openalex.org/W2277195237","https://openalex.org/W2282821441","https://openalex.org/W2321470647","https://openalex.org/W2396881363","https://openalex.org/W2463955103","https://openalex.org/W2506483933","https://openalex.org/W2519656895","https://openalex.org/W2546696630","https://openalex.org/W2550936021","https://openalex.org/W2560730294","https://openalex.org/W2561715562","https://openalex.org/W2563399268","https://openalex.org/W2581887665","https://openalex.org/W2584561145","https://openalex.org/W2592962403","https://openalex.org/W2612131472","https://openalex.org/W2619383789","https://openalex.org/W2623638694","https://openalex.org/W2626792426","https://openalex.org/W2636355936","https://openalex.org/W2736142580","https://openalex.org/W2738406145","https://openalex.org/W2741295496","https://openalex.org/W2750725664","https://openalex.org/W2753738274","https://openalex.org/W2753840835","https://openalex.org/W2759570332","https://openalex.org/W2767249564","https://openalex.org/W2786541991","https://openalex.org/W2787581402","https://openalex.org/W2788810331","https://openalex.org/W2797653144","https://openalex.org/W2799142693","https://openalex.org/W2803125506","https://openalex.org/W2806015799","https://openalex.org/W2807776204","https://openalex.org/W2808359495","https://openalex.org/W2890781596","https://openalex.org/W2890929258","https://openalex.org/W2891444075","https://openalex.org/W2892245540","https://openalex.org/W2894771803","https://openalex.org/W2899448251","https://openalex.org/W2901466771","https://openalex.org/W2902729397","https://openalex.org/W2914217321","https://openalex.org/W2924316106","https://openalex.org/W2937328183","https://openalex.org/W2943038672","https://openalex.org/W2945210081","https://openalex.org/W2946086442","https://openalex.org/W2946200149","https://openalex.org/W2948153973","https://openalex.org/W2949530332","https://openalex.org/W2952577001","https://openalex.org/W2953104586","https://openalex.org/W2955691601","https://openalex.org/W2962772676","https://openalex.org/W2962804981","https://openalex.org/W2962843949","https://openalex.org/W2962850006","https://openalex.org/W2962858109","https://openalex.org/W2962931510","https://openalex.org/W2963063161","https://openalex.org/W2963109634","https://openalex.org/W2963115613","https://openalex.org/W2963224792","https://openalex.org/W2963260436","https://openalex.org/W2963349408","https://openalex.org/W2963349562","https://openalex.org/W2963383024","https://openalex.org/W2963398599","https://openalex.org/W2963518342","https://openalex.org/W2963541336","https://openalex.org/W2963605190","https://openalex.org/W2963606508","https://openalex.org/W2963609017","https://openalex.org/W2963622213","https://openalex.org/W2963717374","https://openalex.org/W2963735856","https://openalex.org/W2963747480","https://openalex.org/W2963870853","https://openalex.org/W2963890755","https://openalex.org/W2963919031","https://openalex.org/W2964010806","https://openalex.org/W2964051877","https://openalex.org/W2964109005","https://openalex.org/W2964118342","https://openalex.org/W2964185501","https://openalex.org/W2964346351","https://openalex.org/W2964349631","https://openalex.org/W2964915587","https://openalex.org/W2964935470","https://openalex.org/W2965116998","https://openalex.org/W2966555834","https://openalex.org/W2967957126","https://openalex.org/W2968095426","https://openalex.org/W2968147594","https://openalex.org/W2969332757","https://openalex.org/W2970231061","https://openalex.org/W2970442950","https://openalex.org/W2970873268","https://openalex.org/W2970972665","https://openalex.org/W2971307358","https://openalex.org/W2972687021","https://openalex.org/W2973126236","https://openalex.org/W2979212481","https://openalex.org/W2979672901","https://openalex.org/W2981851019","https://openalex.org/W2981852735","https://openalex.org/W2982619606","https://openalex.org/W2982625143","https://openalex.org/W2983995706","https://openalex.org/W2987472543","https://openalex.org/W2992433902","https://openalex.org/W2994718079","https://openalex.org/W2996908057","https://openalex.org/W2998577728","https://openalex.org/W3002511705","https://openalex.org/W3014654829","https://openalex.org/W3015591594","https://openalex.org/W3015685572","https://openalex.org/W3022046290","https://openalex.org/W3023989664","https://openalex.org/W3023993913","https://openalex.org/W3024534448","https://openalex.org/W3026092005","https://openalex.org/W3034266838","https://openalex.org/W3034871396","https://openalex.org/W3034972874","https://openalex.org/W3035333188","https://openalex.org/W3035512383","https://openalex.org/W3035574324","https://openalex.org/W3035825181","https://openalex.org/W3036148046","https://openalex.org/W3036595416","https://openalex.org/W3041795964","https://openalex.org/W3048631361","https://openalex.org/W3080935714","https://openalex.org/W3082455399","https://openalex.org/W3088186989","https://openalex.org/W3094175061","https://openalex.org/W3097683561","https://openalex.org/W3100355250","https://openalex.org/W3101065397","https://openalex.org/W3102692100","https://openalex.org/W3104536742","https://openalex.org/W3105009590","https://openalex.org/W3105484484","https://openalex.org/W3106784008","https://openalex.org/W3108655343","https://openalex.org/W3109585842","https://openalex.org/W3110388292","https://openalex.org/W3115266783","https://openalex.org/W3118120400","https://openalex.org/W3118580076","https://openalex.org/W3118966983","https://openalex.org/W3129342828","https://openalex.org/W3133465684","https://openalex.org/W3133702157","https://openalex.org/W3134648203","https://openalex.org/W3136860328","https://openalex.org/W3148388528","https://openalex.org/W3154222058","https://openalex.org/W3161185727","https://openalex.org/W3166100196","https://openalex.org/W3168154341","https://openalex.org/W3171345413","https://openalex.org/W3171884590","https://openalex.org/W3172141633","https://openalex.org/W3174770825","https://openalex.org/W3175304158","https://openalex.org/W3175541806","https://openalex.org/W3176186248","https://openalex.org/W3176724088","https://openalex.org/W3176974280","https://openalex.org/W3177174258","https://openalex.org/W3177934633","https://openalex.org/W3181101419","https://openalex.org/W3182074706","https://openalex.org/W3184144760","https://openalex.org/W3184369217","https://openalex.org/W3193171560","https://openalex.org/W3194633557","https://openalex.org/W3201085013","https://openalex.org/W3203655597","https://openalex.org/W3204047821","https://openalex.org/W3205663647","https://openalex.org/W3205872166","https://openalex.org/W3206316161","https://openalex.org/W3206808041","https://openalex.org/W3211263893","https://openalex.org/W3211495814","https://openalex.org/W3212002421","https://openalex.org/W3212751556","https://openalex.org/W3214127792","https://openalex.org/W4200387604","https://openalex.org/W4205991051","https://openalex.org/W4210261907","https://openalex.org/W4214653736","https://openalex.org/W4214717370","https://openalex.org/W4225323780","https://openalex.org/W4225432580","https://openalex.org/W4229058281","https://openalex.org/W4233762729","https://openalex.org/W4250657332","https://openalex.org/W4283828996","https://openalex.org/W4287113019","https://openalex.org/W4287125738","https://openalex.org/W4288089799","https://openalex.org/W4288096733","https://openalex.org/W4288286281","https://openalex.org/W4292811746","https://openalex.org/W4295951577","https://openalex.org/W4297754112","https://openalex.org/W4297816851","https://openalex.org/W4304014045","https://openalex.org/W4312261477","https://openalex.org/W4312309978","https://openalex.org/W4312407537","https://openalex.org/W4312605942","https://openalex.org/W4312791030","https://openalex.org/W4312810944","https://openalex.org/W4312910992","https://openalex.org/W4312933868","https://openalex.org/W4313164293","https://openalex.org/W4321020678","https://openalex.org/W4376226279","https://openalex.org/W4377825911","https://openalex.org/W4385571053","https://openalex.org/W4386057714","https://openalex.org/W4386290290","https://openalex.org/W4390872723","https://openalex.org/W4390872808","https://openalex.org/W4390873158","https://openalex.org/W6678360021","https://openalex.org/W6679554340","https://openalex.org/W6739811675","https://openalex.org/W6765591853","https://openalex.org/W6777179611","https://openalex.org/W6811886535","https://openalex.org/W6989321458"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W3107602296","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347"],"abstract_inverted_index":{"Multimodal":[0],"machine":[1,61,72,144],"learning":[2,23,62,73],"is":[3,130],"a":[4,113,168],"vibrant":[5],"multi-disciplinary":[6],"research":[7,95,224],"field":[8],"that":[9,161],"aims":[10],"to":[11,70,100,132,206],"design":[12],"computer":[13],"agents":[14],"with":[15],"intelligent":[16],"capabilities":[17],"such":[18,55],"as":[19,56,225],"understanding,":[20,43],"reasoning,":[21],"and":[22,34,49,58,67,81,105,119,125,139,159,166,185,189,210],"through":[24,198],"integrating":[25],"multiple":[26],"communicative":[27],"modalities,":[28],"including":[29],"linguistic,":[30],"acoustic,":[31],"visual,":[32],"tactile,":[33],"physiological":[35],"messages.":[36],"With":[37],"the":[38,71,76,82,89,102,109,137,199,208],"recent":[39,126,190],"interest":[40],"in":[41,52,93,108],"video":[42],"embodied":[44],"autonomous":[45],"agents,":[46],"text-to-image":[47],"generation,":[48],"multisensor":[50],"fusion":[51],"application":[53,117],"domains":[54,118],"healthcare":[57],"robotics,":[59],"multimodal":[60,94,143],"has":[63,96],"brought":[64],"unique":[65],"computational":[66,138],"theoretical":[68,120,140],"challenges":[69],"community":[74],"given":[75],"heterogeneity":[77,155],"of":[78,91,116,136,142,153,170,201],"data":[79],"sources":[80],"interconnections":[83],"often":[84],"found":[85],"between":[86],"modalities.":[87],"However,":[88],"breadth":[90],"progress":[92],"made":[97],"it":[98],"difficult":[99],"identify":[101],"common":[103],"themes":[104],"open":[106,220],"questions":[107],"field.":[110],"By":[111],"synthesizing":[112],"broad":[114],"range":[115],"frameworks":[121],"from":[122],"both":[123],"historical":[124,188],"perspectives,":[127],"this":[128,202],"article":[129],"designed":[131],"provide":[133],"an":[134],"overview":[135],"foundations":[141],"learning.":[145],"We":[146,215],"start":[147],"by":[148,217,227],"defining":[149],"three":[150],"key":[151],"principles":[152],"modality":[154],",":[156,158,176,178,180,182,184],"connections":[157],"interactions":[160],"have":[162],"driven":[163],"subsequent":[164],"innovations,":[165],"propose":[167],"taxonomy":[169],"six":[171],"core":[172],"technical":[173,193],"challenges:":[174],"representation":[175],"alignment":[177],"reasoning":[179],"generation":[181],"transference":[183],"quantification":[186],"covering":[187],"trends.":[191],"Recent":[192],"achievements":[194],"will":[195],"be":[196],"presented":[197],"lens":[200],"taxonomy,":[203],"allowing":[204],"researchers":[205],"understand":[207],"similarities":[209],"differences":[211],"across":[212],"new":[213],"approaches.":[214],"end":[216],"motivating":[218],"several":[219],"problems":[221],"for":[222],"future":[223],"identified":[226],"our":[228],"taxonomy.":[229]},"counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":93},{"year":2024,"cited_by_count":29},{"year":2023,"cited_by_count":1}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}