{"id":"https://openalex.org/W4387446650","doi":"https://doi.org/10.1145/3610661.3617602","title":"Tutorial on Multimodal Machine Learning: Principles, Challenges, and Open Questions","display_name":"Tutorial on Multimodal Machine Learning: Principles, Challenges, and Open Questions","publication_year":2023,"publication_date":"2023-10-09","ids":{"openalex":"https://openalex.org/W4387446650","doi":"https://doi.org/10.1145/3610661.3617602"},"language":"en","primary_location":{"id":"doi:10.1145/3610661.3617602","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3610661.3617602","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Cconference on Multimodal Interaction","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086233510","display_name":"Paul Pu Liang","orcid":"https://orcid.org/0000-0001-7768-3610"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Paul Pu Liang","raw_affiliation_strings":["Machine Learning Department, Carnegie Mellon University, United States of America"],"raw_orcid":"https://orcid.org/0000-0001-7768-3610","affiliations":[{"raw_affiliation_string":"Machine Learning Department, Carnegie Mellon University, United States of America","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081398601","display_name":"Louis\u2010Philippe Morency","orcid":"https://orcid.org/0000-0001-6376-7696"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Louis-Philippe Morency","raw_affiliation_strings":["Language Technologies Institute, Carnegie Mellon University, United States of America"],"raw_orcid":"https://orcid.org/0000-0001-6376-7696","affiliations":[{"raw_affiliation_string":"Language Technologies Institute, Carnegie Mellon University, United States of America","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":46,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"101","last_page":"104"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9908000230789185,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9908000230789185,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9889000058174133,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9775000214576721,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.7020137906074524},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6950353384017944},{"id":"https://openalex.org/keywords/multimodality","display_name":"Multimodality","score":0.6844992637634277},{"id":"https://openalex.org/keywords/multimodal-learning","display_name":"Multimodal learning","score":0.6453967094421387},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.561112642288208},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5583577752113342},{"id":"https://openalex.org/keywords/open-research","display_name":"Open research","score":0.5237950086593628},{"id":"https://openalex.org/keywords/taxonomy","display_name":"Taxonomy (biology)","score":0.49680522084236145},{"id":"https://openalex.org/keywords/multimodal-interaction","display_name":"Multimodal interaction","score":0.4715447723865509},{"id":"https://openalex.org/keywords/embodied-cognition","display_name":"Embodied cognition","score":0.449165403842926},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.44838547706604004},{"id":"https://openalex.org/keywords/interdisciplinarity","display_name":"Interdisciplinarity","score":0.44498422741889954},{"id":"https://openalex.org/keywords/robotics","display_name":"Robotics","score":0.43223071098327637},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.38901734352111816},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.18927541375160217},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.12232309579849243}],"concepts":[{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.7020137906074524},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6950353384017944},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.6844992637634277},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.6453967094421387},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.561112642288208},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5583577752113342},{"id":"https://openalex.org/C2778464652","wikidata":"https://www.wikidata.org/wiki/Q309849","display_name":"Open research","level":2,"score":0.5237950086593628},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.49680522084236145},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.4715447723865509},{"id":"https://openalex.org/C100609095","wikidata":"https://www.wikidata.org/wiki/Q1335050","display_name":"Embodied cognition","level":2,"score":0.449165403842926},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.44838547706604004},{"id":"https://openalex.org/C133741464","wikidata":"https://www.wikidata.org/wiki/Q849359","display_name":"Interdisciplinarity","level":2,"score":0.44498422741889954},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.43223071098327637},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.38901734352111816},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.18927541375160217},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.12232309579849243},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3610661.3617602","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3610661.3617602","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Cconference on Multimodal Interaction","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7200000286102295}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W245846980","https://openalex.org/W1529533208","https://openalex.org/W1773149199","https://openalex.org/W1882423120","https://openalex.org/W2038227658","https://openalex.org/W2048679005","https://openalex.org/W2797653144","https://openalex.org/W2803125506","https://openalex.org/W2945210081","https://openalex.org/W2963115613","https://openalex.org/W2963349562","https://openalex.org/W2964118342","https://openalex.org/W2964915587","https://openalex.org/W3002511705","https://openalex.org/W3034211329","https://openalex.org/W3035333188","https://openalex.org/W3035825181","https://openalex.org/W3036595416","https://openalex.org/W3100801259","https://openalex.org/W3110388292","https://openalex.org/W3157481066","https://openalex.org/W3158986867","https://openalex.org/W3175541806","https://openalex.org/W3176974280","https://openalex.org/W3177934633","https://openalex.org/W3204047821","https://openalex.org/W4230074614","https://openalex.org/W4312791030","https://openalex.org/W4312933868"],"related_works":["https://openalex.org/W2968820833","https://openalex.org/W3093803775","https://openalex.org/W2068964259","https://openalex.org/W2013013717","https://openalex.org/W66256988","https://openalex.org/W4302857920","https://openalex.org/W1987340722","https://openalex.org/W199123384","https://openalex.org/W2739856905","https://openalex.org/W4239305747"],"abstract_inverted_index":{"Multimodal":[0],"machine":[1,63,135],"learning":[2,20,64,167],"is":[3,121,162,176,198],"a":[4,104,139,179],"vibrant":[5],"multi-disciplinary":[6],"research":[7,86,204],"field":[8],"that":[9],"aims":[10],"to":[11,61,91,123],"design":[12],"computer":[13],"agents":[14],"capable":[15],"of":[16,69,82,107,127,133,142,181],"understanding,":[17,40],"reasoning,":[18],"and":[19,31,46,51,58,72,96,110,116,130,149,171,195],"through":[21],"integrating":[22],"multiple":[23],"communicative":[24],"modalities,":[25],"including":[26],"linguistic,":[27],"acoustic,":[28],"visual,":[29],"tactile,":[30],"physiological":[32],"messages.":[33],"With":[34],"the":[35,62,67,73,80,93,100,128,164],"recent":[36,117],"interest":[37],"in":[38,49,84,99,166,187,192],"video":[39],"embodied":[41],"autonomous":[42],"agents,":[43],"text-to-image":[44],"generation,":[45],"multisensor":[47],"fusion":[48],"healthcare":[50],"robotics,":[52],"multimodality":[53],"has":[54,87],"brought":[55],"unique":[56],"computational":[57,129],"theoretical":[59,111,131],"challenges":[60,185],"community":[65],"given":[66],"heterogeneity":[68],"data":[70],"sources":[71],"interconnections":[74],"often":[75],"found":[76],"between":[77],"modalities.":[78],"However,":[79],"breadth":[81],"progress":[83],"multimodal":[85,134,147,188],"made":[88],"it":[89,177],"difficult":[90],"identify":[92],"common":[94],"themes":[95],"open":[97],"questions":[98],"field.":[101],"By":[102],"synthesizing":[103],"broad":[105],"range":[106],"application":[108],"domains":[109],"frameworks":[112],"from":[113,168],"both":[114],"historical":[115],"perspectives,":[118],"this":[119,154],"tutorial":[120,155],"designed":[122],"provide":[124],"an":[125],"overview":[126],"foundations":[132],"learning.":[136],"Building":[137],"upon":[138],"new":[140],"edition":[141],"our":[143,208],"survey":[144],"paper":[145],"on":[146],"ML":[148,189],"academic":[150],"courses":[151],"at":[152],"CMU,":[153],"will":[156],"cover":[157],"three":[158],"topics:":[159],"(1)":[160],"what":[161,197],"multimodal:":[163],"principles":[165],"heterogeneous,":[169],"connected,":[170],"interacting":[172],"data,":[173],"(2)":[174],"why":[175],"hard:":[178],"taxonomy":[180],"six":[182],"core":[183],"technical":[184],"faced":[186],"but":[190],"understudied":[191],"unimodal":[193],"ML,":[194],"(3)":[196],"next:":[199],"major":[200],"directions":[201],"for":[202],"future":[203],"as":[205],"identified":[206],"by":[207],"taxonomy.":[209]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":14},{"year":2023,"cited_by_count":13}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}