{"id":"https://openalex.org/W3093763606","doi":"https://doi.org/10.1145/3382507.3418887","title":"Toward Multimodal Modeling of Emotional Expressiveness","display_name":"Toward Multimodal Modeling of Emotional Expressiveness","publication_year":2020,"publication_date":"2020-10-21","ids":{"openalex":"https://openalex.org/W3093763606","doi":"https://doi.org/10.1145/3382507.3418887","mag":"3093763606","pmid":"https://pubmed.ncbi.nlm.nih.gov/33969360"},"language":"en","primary_location":{"id":"doi:10.1145/3382507.3418887","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3382507.3418887","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3382507.3418887","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 International Conference on Multimodal Interaction","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3382507.3418887","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5004184442","display_name":"Victoria Lin","orcid":"https://orcid.org/0000-0003-2873-2177"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Victoria Lin","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078452888","display_name":"Jeffrey M. Girard","orcid":"https://orcid.org/0000-0002-7359-3746"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jeffrey M. Girard","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054617714","display_name":"Michael A. Sayette","orcid":"https://orcid.org/0000-0001-7617-5198"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael A. Sayette","raw_affiliation_strings":["University of Pittsburgh, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pittsburgh, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I170201317"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081398601","display_name":"Louis\u2010Philippe Morency","orcid":"https://orcid.org/0000-0001-6376-7696"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Louis-Philippe Morency","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.1721,"has_fulltext":true,"cited_by_count":8,"citation_normalized_percentile":{"value":0.80576853,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"2020","issue":null,"first_page":"548","last_page":"557"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11094","display_name":"Face Recognition and Perception","score":0.9847000241279602,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9829999804496765,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.6058732867240906},{"id":"https://openalex.org/keywords/emotional-expression","display_name":"Emotional expression","score":0.5866097807884216},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5517293810844421},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.5448777675628662},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.5240818858146667},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.45230889320373535},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4033662676811218},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3781370222568512}],"concepts":[{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.6058732867240906},{"id":"https://openalex.org/C143110190","wikidata":"https://www.wikidata.org/wiki/Q5373787","display_name":"Emotional expression","level":2,"score":0.5866097807884216},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5517293810844421},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.5448777675628662},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.5240818858146667},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.45230889320373535},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4033662676811218},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3781370222568512},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3382507.3418887","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3382507.3418887","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3382507.3418887","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 International Conference on Multimodal Interaction","raw_type":"proceedings-article"},{"id":"pmid:33969360","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/33969360","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ... ACM International Conference on Multimodal Interaction. ICMI (Conference)","raw_type":"Journal Article"},{"id":"pmh:oai:pubmedcentral.nih.gov:8106384","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/8106384","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proc ACM Int Conf Multimodal Interact","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1145/3382507.3418887","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3382507.3418887","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3382507.3418887","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 International Conference on Multimodal Interaction","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1133040973","display_name":null,"funder_award_id":"R01 MH096951","funder_id":"https://openalex.org/F4320337346","funder_display_name":"National Institute of Mental Health"},{"id":"https://openalex.org/G2570099219","display_name":"NCS-FO:Collaborative Research:Decoding and Reconstructing the Neural Basis of Real World Social Perception","funder_award_id":"1734868","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2718465150","display_name":null,"funder_award_id":"U01 MH116923","funder_id":"https://openalex.org/F4320337346","funder_display_name":"National Institute of Mental Health"},{"id":"https://openalex.org/G3869343000","display_name":null,"funder_award_id":"U01MH116925","funder_id":"https://openalex.org/F4320337346","funder_display_name":"National Institute of Mental Health"},{"id":"https://openalex.org/G574185058","display_name":"SCH: INT: Collaborative Research: Dyadic Behavior Informatics for Psychotherapy Process and Outcome","funder_award_id":"1722822","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6078575908","display_name":"Automatic Multimodal Affect Detection for Research and Clinical Use\u00a0","funder_award_id":"5r01mh096951-10","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G875270525","display_name":null,"funder_award_id":"R01AA015773","funder_id":"https://openalex.org/F4320337330","funder_display_name":"National Institute on Alcohol Abuse and Alcoholism"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320337330","display_name":"National Institute on Alcohol Abuse and Alcoholism","ror":"https://ror.org/02jzrsm59"},{"id":"https://openalex.org/F4320337346","display_name":"National Institute of Mental Health","ror":"https://ror.org/04xeg9z08"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3093763606.pdf","grobid_xml":"https://content.openalex.org/works/W3093763606.grobid-xml"},"referenced_works_count":71,"referenced_works":["https://openalex.org/W55356581","https://openalex.org/W202727379","https://openalex.org/W606553535","https://openalex.org/W613277099","https://openalex.org/W1494186755","https://openalex.org/W1665332082","https://openalex.org/W1773917465","https://openalex.org/W1963560988","https://openalex.org/W1974752902","https://openalex.org/W1984806222","https://openalex.org/W1995945562","https://openalex.org/W2019390009","https://openalex.org/W2022060646","https://openalex.org/W2025905516","https://openalex.org/W2028422222","https://openalex.org/W2056030034","https://openalex.org/W2062118960","https://openalex.org/W2067724039","https://openalex.org/W2069864193","https://openalex.org/W2071666535","https://openalex.org/W2080507775","https://openalex.org/W2089809964","https://openalex.org/W2100070324","https://openalex.org/W2100348682","https://openalex.org/W2108445559","https://openalex.org/W2109626108","https://openalex.org/W2110673279","https://openalex.org/W2118338381","https://openalex.org/W2122825543","https://openalex.org/W2137226992","https://openalex.org/W2137702359","https://openalex.org/W2138804008","https://openalex.org/W2146910990","https://openalex.org/W2148071321","https://openalex.org/W2153597356","https://openalex.org/W2154716422","https://openalex.org/W2155541015","https://openalex.org/W2156503193","https://openalex.org/W2165312226","https://openalex.org/W2177847511","https://openalex.org/W2243226955","https://openalex.org/W2327037637","https://openalex.org/W2394723605","https://openalex.org/W2403817006","https://openalex.org/W2410202994","https://openalex.org/W2573933330","https://openalex.org/W2592574207","https://openalex.org/W2610961739","https://openalex.org/W2734289922","https://openalex.org/W2799041689","https://openalex.org/W2805489729","https://openalex.org/W2807126412","https://openalex.org/W2888683367","https://openalex.org/W2901335288","https://openalex.org/W2905327414","https://openalex.org/W2912483755","https://openalex.org/W2934872584","https://openalex.org/W2943755647","https://openalex.org/W2950510190","https://openalex.org/W2951914964","https://openalex.org/W2959143276","https://openalex.org/W3005563669","https://openalex.org/W3018219061","https://openalex.org/W3104819369","https://openalex.org/W4210957923","https://openalex.org/W4212774754","https://openalex.org/W4252792238","https://openalex.org/W4253718710","https://openalex.org/W4296979447","https://openalex.org/W4403242122","https://openalex.org/W6629853029"],"related_works":["https://openalex.org/W2385859805","https://openalex.org/W2530972254","https://openalex.org/W2374013449","https://openalex.org/W73545470","https://openalex.org/W2364381299","https://openalex.org/W2374430585","https://openalex.org/W3144423903","https://openalex.org/W2377397762","https://openalex.org/W2793967660","https://openalex.org/W2361654993"],"abstract_inverted_index":{"Emotional":[0],"expressiveness":[1,24,46,70,118],"captures":[2],"the":[3,19,31,40,88,159,168],"extent":[4],"to":[5,10,18,42,48,87,100,119,128,151,163,200],"which":[6,94],"a":[7,179],"person":[8],"tends":[9],"outwardly":[11],"display":[12],"their":[13],"emotions":[14],"through":[15],"behavior.":[16],"Due":[17],"close":[20],"relationship":[21],"between":[22],"emotional":[23,45,69,91,101,117,205],"and":[25,54,76,112,124,131,155,183,195],"behavioral":[26,78,85,95,185],"health,":[27],"as":[28,30,187],"well":[29,67],"crucial":[32],"role":[33],"that":[34],"it":[35],"plays":[36],"in":[37,51],"social":[38,201],"interaction,":[39],"ability":[41],"automatically":[43],"predict":[44,204],"stands":[47],"spur":[49],"advances":[50],"science,":[52],"medicine,":[53],"industry.":[55],"In":[56],"this":[57,126,143],"paper,":[58],"we":[59,107],"explore":[60],"three":[61],"related":[62,99,199],"research":[63],"questions.":[64],"First,":[65],"how":[66,81],"can":[68],"be":[71],"predicted":[72],"from":[73],"visual,":[74],"linguistic,":[75],"multimodal":[77],"signals?":[79],"Second,":[80],"important":[82],"is":[83],"each":[84],"modality":[86,161],"prediction":[89],"of":[90,115,173,181,197],"expressiveness?":[92,102],"Third,":[93],"signals":[96],"are":[97],"reliably":[98,203],"To":[103],"answer":[104],"these":[105],"questions,":[106],"add":[108],"highly":[109],"reliable":[110],"transcripts":[111],"human":[113],"ratings":[114],"perceived":[116],"an":[120],"existing":[121],"video":[122],"database":[123],"use":[125,196],"data":[127],"train,":[129],"validate,":[130],"test":[132],"predictive":[133,140],"models.":[134],"Our":[135],"best":[136,153],"model":[137],"shows":[138],"promising":[139],"performance":[141],"on":[142,158,167],"dataset":[144],"(RMSE=0.65,":[145],"R^2=0.45,":[146],"r=0.74).":[147],"Multimodal":[148],"models":[149,156,165],"tend":[150,162],"perform":[152],"overall,":[154],"trained":[157,166],"linguistic":[160,184],"outperform":[164],"visual":[169,182],"modality.":[170],"Finally,":[171],"examination":[172],"our":[174],"interpretable":[175],"models'":[176],"coefficients":[177],"reveals":[178],"number":[180],"signals---such":[186],"facial":[188],"action":[189],"unit":[190],"intensity,":[191],"overall":[192],"word":[193],"count,":[194],"words":[198],"processes---that":[202],"expressiveness.":[206]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-10-10T00:00:00"}