{"id":"https://openalex.org/W4387421643","doi":"https://doi.org/10.1145/3577190.3614151","title":"Multimodal Fusion Interactions: A Study of Human and Automatic Quantification","display_name":"Multimodal Fusion Interactions: A Study of Human and Automatic Quantification","publication_year":2023,"publication_date":"2023-10-07","ids":{"openalex":"https://openalex.org/W4387421643","doi":"https://doi.org/10.1145/3577190.3614151"},"language":"en","primary_location":{"id":"doi:10.1145/3577190.3614151","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3577190.3614151","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3577190.3614151","source":{"id":"https://openalex.org/S4363608440","display_name":"INTERNATIONAL CONFERENCE ON MULTIMODAL INTERACTION","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"INTERNATIONAL CONFERENCE ON MULTIMODAL INTERACTION","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3577190.3614151","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086233510","display_name":"Paul Pu Liang","orcid":"https://orcid.org/0000-0001-7768-3610"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Paul Pu Liang","raw_affiliation_strings":["Machine Learning Department, Carnegie Mellon University, United States"],"raw_orcid":"https://orcid.org/0000-0001-7768-3610","affiliations":[{"raw_affiliation_string":"Machine Learning Department, Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048386360","display_name":"Yun Cheng","orcid":"https://orcid.org/0009-0009-8916-6667"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yun Cheng","raw_affiliation_strings":["Carnegie Mellon University, United States"],"raw_orcid":"https://orcid.org/0009-0009-8916-6667","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071983998","display_name":"Ruslan Salakhutdinov","orcid":"https://orcid.org/0000-0002-3752-2756"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ruslan Salakhutdinov","raw_affiliation_strings":["Carnegie Mellon University, United States"],"raw_orcid":"https://orcid.org/0000-0002-3752-2756","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081398601","display_name":"Louis\u2010Philippe Morency","orcid":"https://orcid.org/0000-0001-6376-7696"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Louis-Philippe Morency","raw_affiliation_strings":["Language Technologies Institute, Carnegie Mellon University, United States"],"raw_orcid":"https://orcid.org/0000-0001-6376-7696","affiliations":[{"raw_affiliation_string":"Language Technologies Institute, Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"425","last_page":"435"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9926000237464905,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.8446181416511536},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8324683904647827},{"id":"https://openalex.org/keywords/counterfactual-thinking","display_name":"Counterfactual thinking","score":0.7546178698539734},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.6717668771743774},{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.6140715479850769},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.607161283493042},{"id":"https://openalex.org/keywords/decomposition","display_name":"Decomposition","score":0.4617280066013336},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.44857466220855713},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3942358195781708},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3812558054924011}],"concepts":[{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.8446181416511536},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8324683904647827},{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.7546178698539734},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.6717668771743774},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.6140715479850769},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.607161283493042},{"id":"https://openalex.org/C124681953","wikidata":"https://www.wikidata.org/wiki/Q339062","display_name":"Decomposition","level":2,"score":0.4617280066013336},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.44857466220855713},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3942358195781708},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3812558054924011},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3577190.3614151","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3577190.3614151","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3577190.3614151","source":{"id":"https://openalex.org/S4363608440","display_name":"INTERNATIONAL CONFERENCE ON MULTIMODAL INTERACTION","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"INTERNATIONAL CONFERENCE ON MULTIMODAL INTERACTION","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3577190.3614151","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3577190.3614151","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3577190.3614151","source":{"id":"https://openalex.org/S4363608440","display_name":"INTERNATIONAL CONFERENCE ON MULTIMODAL INTERACTION","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"INTERNATIONAL CONFERENCE ON MULTIMODAL INTERACTION","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2384485762","display_name":null,"funder_award_id":"R01MH125740, R01MH132225, R01MH096951, R21MH130767","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G3550935145","display_name":null,"funder_award_id":"N000141812861","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G4340203621","display_name":"CAREER: Learning Nonverbal Signatures","funder_award_id":"1750439","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5103598966","display_name":null,"funder_award_id":"FA87502321015","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G574185058","display_name":"SCH: INT: Collaborative Research: Dyadic Behavior Informatics for Psychotherapy Process and Outcome","funder_award_id":"1722822","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6353494076","display_name":null,"funder_award_id":"N000142312368","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G7353603994","display_name":null,"funder_award_id":"1722822, 1750439","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7421738053","display_name":null,"funder_award_id":"R21MH130767","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G8792181326","display_name":null,"funder_award_id":"N000141812861, N000142312368","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320309480","display_name":"Nvidia","ror":"https://ror.org/03jdj4y14"},{"id":"https://openalex.org/F4320310207","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33"},{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"},{"id":"https://openalex.org/F4320337968","display_name":"Center for Machine Learning and Health, School of Computer Science, Carnegie Mellon University","ror":null},{"id":"https://openalex.org/F4320338294","display_name":"Air Force Research Laboratory","ror":"https://ror.org/02e2egq70"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387421643.pdf","grobid_xml":"https://content.openalex.org/works/W4387421643.grobid-xml"},"referenced_works_count":43,"referenced_works":["https://openalex.org/W245846980","https://openalex.org/W1493603593","https://openalex.org/W1582211066","https://openalex.org/W1882423120","https://openalex.org/W1964891720","https://openalex.org/W1965680102","https://openalex.org/W1967259204","https://openalex.org/W2028975938","https://openalex.org/W2037667080","https://openalex.org/W2048679005","https://openalex.org/W2081988641","https://openalex.org/W2083638007","https://openalex.org/W2139708568","https://openalex.org/W2155279768","https://openalex.org/W2163070219","https://openalex.org/W2164699598","https://openalex.org/W2176910270","https://openalex.org/W2290452516","https://openalex.org/W2560730294","https://openalex.org/W2561715562","https://openalex.org/W2616141209","https://openalex.org/W2753175802","https://openalex.org/W2883409523","https://openalex.org/W2886193235","https://openalex.org/W2937328183","https://openalex.org/W2963349408","https://openalex.org/W2964010806","https://openalex.org/W2970217405","https://openalex.org/W3002511705","https://openalex.org/W3006449229","https://openalex.org/W3011336756","https://openalex.org/W3035145964","https://openalex.org/W3098439442","https://openalex.org/W3100801259","https://openalex.org/W3104536742","https://openalex.org/W3105841952","https://openalex.org/W3135367836","https://openalex.org/W3176463841","https://openalex.org/W3204047821","https://openalex.org/W4288096733","https://openalex.org/W4297825594","https://openalex.org/W4299828299","https://openalex.org/W4385573973"],"related_works":["https://openalex.org/W73545470","https://openalex.org/W4224266612","https://openalex.org/W2383394264","https://openalex.org/W4320153225","https://openalex.org/W4293261942","https://openalex.org/W3125968744","https://openalex.org/W203959209","https://openalex.org/W2167701463","https://openalex.org/W2110287964","https://openalex.org/W4307407935"],"abstract_inverted_index":{"In":[0,36],"order":[1],"to":[2,11,87,119,132,148,154,184,193],"perform":[3,40],"multimodal":[4,51,204],"fusion":[5],"of":[6,33,44,50,115,177,188],"heterogeneous":[7],"signals,":[8],"we":[9,39,171],"need":[10],"understand":[12],"their":[13,92],"interactions:":[14,52],"how":[15,26,45,91],"each":[16,178],"modality":[17,83,135],"individually":[18,122],"provides":[19],"information":[20,28,108,194],"useful":[21],"for":[22,202],"a":[23,41,137,156,182],"task":[24],"and":[25,66,69,123,144,169,175,180,190,199],"this":[27,37],"changes":[29,94],"in":[30],"the":[31,60,63,74,78,81,97,113,117,126,130,140,146],"presence":[32],"other":[34,141],"modalities.":[35,166],"paper,":[38],"comparative":[42],"study":[43],"humans":[46],"annotate":[47,59,112],"two":[48],"categorizations":[49],"(1)":[53],"partial":[54,189],"labels,":[55,72],"where":[56,73,110],"different":[57],"annotators":[58,111],"label":[61,79],"given":[62,80,96],"first,":[64],"second,":[65],"both":[67,150],"modalities,":[68],"(2)":[70],"counterfactual":[71,191],"same":[75,127],"annotator":[76],"annotates":[77],"first":[82],"before":[84],"asking":[85],"them":[86],"explicitly":[88],"reason":[89],"about":[90],"answer":[93],"when":[95],"second.":[98],"We":[99],"further":[100],"propose":[101,181],"an":[102,197],"alternative":[103],"taxonomy":[104],"based":[105],"on":[106],"(3)":[107],"decomposition,":[109,195],"degrees":[114],"redundancy:":[116],"extent":[118,131,147],"which":[120,133,149],"modalities":[121,151],"together":[124],"give":[125],"predictions,":[128],"uniqueness:":[129],"one":[134,153,159],"enables":[136],"prediction":[138,157],"that":[139,158],"does":[142],"not,":[143],"synergy:":[145],"enable":[152],"make":[155,163],"would":[160],"not":[161],"otherwise":[162],"using":[164],"individual":[165],"Through":[167],"experiments":[168],"annotations,":[170],"highlight":[172],"several":[173],"opportunities":[174],"limitations":[176],"approach":[179],"method":[183,201],"automatically":[185],"convert":[186],"annotations":[187],"labels":[192],"yielding":[196],"accurate":[198],"efficient":[200],"quantifying":[203],"interactions.":[205]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}