{"id":"https://openalex.org/W4403913146","doi":"https://doi.org/10.1145/3678957.3689332","title":"Multimodal Emotion Recognition Harnessing the Complementarity of Speech, Language, and Vision","display_name":"Multimodal Emotion Recognition Harnessing the Complementarity of Speech, Language, and Vision","publication_year":2024,"publication_date":"2024-10-30","ids":{"openalex":"https://openalex.org/W4403913146","doi":"https://doi.org/10.1145/3678957.3689332"},"language":"en","primary_location":{"id":"doi:10.1145/3678957.3689332","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3678957.3689332","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Conference on Multimodel Interaction","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3678957.3689332","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040324682","display_name":"Thomas Thebaud","orcid":"https://orcid.org/0000-0001-8953-7872"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210098728","display_name":"Language Computer (United States)","ror":"https://ror.org/013mgh495","country_code":"US","type":"company","lineage":["https://openalex.org/I4210098728"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Thomas Thebaud","raw_affiliation_strings":["Center for Language and Speech Processing, Johns Hopkins University, USA and Human Language Technology Center of Excellence, Johns Hopkins University, USA"],"raw_orcid":"https://orcid.org/0000-0001-8953-7872","affiliations":[{"raw_affiliation_string":"Center for Language and Speech Processing, Johns Hopkins University, USA and Human Language Technology Center of Excellence, Johns Hopkins University, USA","institution_ids":["https://openalex.org/I4210098728","https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035357020","display_name":"Anna Favaro","orcid":"https://orcid.org/0000-0001-8159-9841"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anna Favaro","raw_affiliation_strings":["Center for Language and Speech Processing, Johns Hopkins University, USA"],"raw_orcid":"https://orcid.org/0000-0001-8159-9841","affiliations":[{"raw_affiliation_string":"Center for Language and Speech Processing, Johns Hopkins University, USA","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056154248","display_name":"Yutong Guan","orcid":null},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yaohan Guan","raw_affiliation_strings":["Center for Language and Speech Processing, Johns Hopkins University, USA"],"raw_orcid":"https://orcid.org/0009-0007-6263-3232","affiliations":[{"raw_affiliation_string":"Center for Language and Speech Processing, Johns Hopkins University, USA","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101073343","display_name":"Yuchen Yang","orcid":"https://orcid.org/0009-0001-7697-5184"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuchen Yang","raw_affiliation_strings":["Center for Language and Speech Processing, Johns Hopkins University, USA"],"raw_orcid":"https://orcid.org/0009-0001-7697-5184","affiliations":[{"raw_affiliation_string":"Center for Language and Speech Processing, Johns Hopkins University, USA","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031769828","display_name":"Prabhav Singh","orcid":"https://orcid.org/0000-0003-3779-3588"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Prabhav Singh","raw_affiliation_strings":["Center for Language and Speech Processing, Johns Hopkins University, USA"],"raw_orcid":"https://orcid.org/0000-0003-3779-3588","affiliations":[{"raw_affiliation_string":"Center for Language and Speech Processing, Johns Hopkins University, USA","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057010207","display_name":"Jes\u00fas Villalba","orcid":"https://orcid.org/0000-0001-9459-8426"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210098728","display_name":"Language Computer (United States)","ror":"https://ror.org/013mgh495","country_code":"US","type":"company","lineage":["https://openalex.org/I4210098728"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jesus Villalba","raw_affiliation_strings":["Center for Language and Speech Processing, Johns Hopkins University, USA and Human Language Technology Center of Excellence, Johns Hopkins University, USA"],"raw_orcid":"https://orcid.org/0000-0001-9459-8426","affiliations":[{"raw_affiliation_string":"Center for Language and Speech Processing, Johns Hopkins University, USA and Human Language Technology Center of Excellence, Johns Hopkins University, USA","institution_ids":["https://openalex.org/I4210098728","https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069488212","display_name":"Laureano Moro-Vel\u00e1zquez","orcid":"https://orcid.org/0000-0002-3033-7005"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210098728","display_name":"Language Computer (United States)","ror":"https://ror.org/013mgh495","country_code":"US","type":"company","lineage":["https://openalex.org/I4210098728"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Laureano Mono-Velazquez","raw_affiliation_strings":["Center for Language and Speech Processing, Johns Hopkins University, USA and Human Language Technology Center of Excellence, Johns Hopkins University, USA"],"raw_orcid":"https://orcid.org/0000-0002-3033-7005","affiliations":[{"raw_affiliation_string":"Center for Language and Speech Processing, Johns Hopkins University, USA and Human Language Technology Center of Excellence, Johns Hopkins University, USA","institution_ids":["https://openalex.org/I4210098728","https://openalex.org/I145311948"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5050632169","display_name":"Najim Dehak","orcid":"https://orcid.org/0000-0002-4489-5753"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210098728","display_name":"Language Computer (United States)","ror":"https://ror.org/013mgh495","country_code":"US","type":"company","lineage":["https://openalex.org/I4210098728"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Najim Dehak","raw_affiliation_strings":["Center for Language and Speech Processing, Johns Hopkins University, USA and Human Language Technology Center of Excellence, Johns Hopkins University, USA"],"raw_orcid":"https://orcid.org/0000-0002-4489-5753","affiliations":[{"raw_affiliation_string":"Center for Language and Speech Processing, Johns Hopkins University, USA and Human Language Technology Center of Excellence, Johns Hopkins University, USA","institution_ids":["https://openalex.org/I4210098728","https://openalex.org/I145311948"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"684","last_page":"689"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9952999949455261,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/complementarity","display_name":"Complementarity (molecular biology)","score":0.8280975818634033},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6518759727478027},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.6357214450836182},{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"Emotion recognition","score":0.6246806383132935},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3983568549156189},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.38532835245132446},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.33409827947616577}],"concepts":[{"id":"https://openalex.org/C202269582","wikidata":"https://www.wikidata.org/wiki/Q2644277","display_name":"Complementarity (molecular biology)","level":2,"score":0.8280975818634033},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6518759727478027},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6357214450836182},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.6246806383132935},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3983568549156189},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38532835245132446},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.33409827947616577},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3678957.3689332","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3678957.3689332","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Conference on Multimodel Interaction","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3678957.3689332","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3678957.3689332","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Conference on Multimodel Interaction","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W2064675550","https://openalex.org/W2890964092","https://openalex.org/W2963839617","https://openalex.org/W2964350391","https://openalex.org/W2970641574","https://openalex.org/W2986154550","https://openalex.org/W3006926732","https://openalex.org/W3015707499","https://openalex.org/W3036601975","https://openalex.org/W3103187652","https://openalex.org/W3128513378","https://openalex.org/W3209984917","https://openalex.org/W4287119707","https://openalex.org/W4296068413","https://openalex.org/W4312976151","https://openalex.org/W4387580450","https://openalex.org/W4391021627","https://openalex.org/W4399196337"],"related_works":["https://openalex.org/W2521519254","https://openalex.org/W3139833644","https://openalex.org/W3123110765","https://openalex.org/W4383553409","https://openalex.org/W2104948296","https://openalex.org/W1735800226","https://openalex.org/W4285172739","https://openalex.org/W3123208392","https://openalex.org/W3126677997","https://openalex.org/W1610857240"],"abstract_inverted_index":{"In":[0],"the":[1,31,73,83,106,114],"realm":[2],"of":[3,17,75,108,116,120],"audiovisual":[4],"emotion":[5],"recognition,":[6],"a":[7],"significant":[8],"challenge":[9,84],"lies":[10],"in":[11,54,78,131],"developing":[12],"neural":[13],"network":[14],"architectures":[15],"capable":[16],"effectively":[18],"harnessing":[19],"and":[20,41,59,69,99,101,129],"integrating":[21,109],"multimodal":[22,90],"information.":[23],"This":[24,112],"study":[25],"introduces":[26],"an":[27],"advanced":[28],"methodology":[29],"for":[30,56],"Empathic":[32],"Virtual":[33],"Agent":[34],"Challenge":[35],"(EVAC),":[36],"utilizing":[37],"state-of-the-art":[38],"speech,":[39],"language,":[40],"image":[42],"models.":[43],"Specifically,":[44],"we":[45,71],"leverage":[46],"cutting-edge":[47],"pre-trained":[48],"models,":[49],"including":[50],"multilingual":[51],"variants":[52],"fine-tuned":[53],"French":[55],"each":[57],"modality,":[58],"integrate":[60],"them":[61],"using":[62],"late":[63],"fusion":[64],"techniques.":[65],"Through":[66],"extensive":[67],"experimentation":[68],"validation,":[70],"demonstrate":[72],"efficacy":[74],"our":[76],"approach":[77],"achieving":[79],"competitive":[80],"results":[81],"on":[82],"dataset.":[85],"Our":[86],"findings":[87],"highlight":[88],"that":[89],"approaches":[91],"outperform":[92],"unimodal":[93],"methods":[94],"across":[95],"Core":[96],"Affect":[97],"Presence":[98],"Intensity":[100],"Appraisal":[102],"Dimensions":[103],"tasks,":[104],"underscoring":[105],"effectiveness":[107],"diverse":[110],"modalities.":[111],"underscores":[113],"importance":[115],"leveraging":[117],"multiple":[118],"sources":[119],"information":[121],"to":[122],"capture":[123],"nuanced":[124],"emotional":[125],"states":[126],"more":[127],"accurately":[128],"robustly":[130],"real-world":[132],"applications.":[133]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}