{"id":"https://openalex.org/W4387969701","doi":"https://doi.org/10.1145/3581783.3613846","title":"Unlocking the Power of Multimodal Learning for Emotion Recognition in Conversation","display_name":"Unlocking the Power of Multimodal Learning for Emotion Recognition in Conversation","publication_year":2023,"publication_date":"2023-10-26","ids":{"openalex":"https://openalex.org/W4387969701","doi":"https://doi.org/10.1145/3581783.3613846"},"language":"en","primary_location":{"id":"doi:10.1145/3581783.3613846","doi":"https://doi.org/10.1145/3581783.3613846","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3581783.3613846","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3613846","source":{"id":"https://openalex.org/S4363608757","display_name":"ACM International Conference on Multimedia (ACM MM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Multimedia","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3613846","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103130904","display_name":"Yunxiao Wang","orcid":"https://orcid.org/0000-0003-1897-6557","observed_orcids":["https://orcid.org/0000-0003-1897-6557"]},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yunxiao Wang","raw_affiliation_strings":["Shandong University, Jinan, China"],"raw_orcid":"https://orcid.org/0000-0003-1897-6557","affiliations":[{"raw_affiliation_string":"Shandong University, Jinan, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100457438","display_name":"Meng Liu","orcid":"https://orcid.org/0000-0002-1582-5764","observed_orcids":["https://orcid.org/0000-0002-1582-5764"]},"institutions":[{"id":"https://openalex.org/I44445938","display_name":"Shandong Jianzhu University","ror":"https://ror.org/01gbfax37","country_code":"CN","type":"education","lineage":["https://openalex.org/I44445938"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Meng Liu","raw_affiliation_strings":["Shandong Jianzhu University, Jinan, China"],"raw_orcid":"https://orcid.org/0000-0002-1582-5764","affiliations":[{"raw_affiliation_string":"Shandong Jianzhu University, Jinan, China","institution_ids":["https://openalex.org/I44445938"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102905093","display_name":"Zhe Li","orcid":"https://orcid.org/0009-0008-4370-6616","observed_orcids":["https://orcid.org/0009-0008-4370-6616"]},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhe Li","raw_affiliation_strings":["Shandong University, Jinan, China"],"raw_orcid":"https://orcid.org/0009-0008-4370-6616","affiliations":[{"raw_affiliation_string":"Shandong University, Jinan, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101875643","display_name":"Yupeng Hu","orcid":"https://orcid.org/0000-0002-5653-8286","observed_orcids":["https://orcid.org/0000-0002-5653-8286","https://orcid.org/0009-0002-8349-1513","https://orcid.org/0000-0003-1692-530X"]},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yupeng Hu","raw_affiliation_strings":["Shandong University, Jinan, China"],"raw_orcid":"https://orcid.org/0000-0002-5653-8286","affiliations":[{"raw_affiliation_string":"Shandong University, Jinan, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052168662","display_name":"Xin Luo","orcid":"https://orcid.org/0000-0002-6901-5476","observed_orcids":["https://orcid.org/0000-0002-6901-5476"]},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Luo","raw_affiliation_strings":["Shandong University, Jinan, China"],"raw_orcid":"https://orcid.org/0000-0002-6901-5476","affiliations":[{"raw_affiliation_string":"Shandong University, Jinan, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5038612499","display_name":"Liqiang Nie","orcid":"https://orcid.org/0000-0003-1476-0273","observed_orcids":["https://orcid.org/0000-0003-1476-0273"]},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liqiang Nie","raw_affiliation_strings":["Harbin Institute of Technology (Shenzhen), Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0003-1476-0273","affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology (Shenzhen), Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.9885,"has_fulltext":true,"cited_by_count":18,"citation_normalized_percentile":{"value":0.9041639,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"5947","last_page":"5955"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.961899995803833,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.961899995803833,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.02850000001490116,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.004699999932199717,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/emotion-recognition-in-conversation","display_name":"emotion recognition in conversation","score":0.9399999976158142},{"id":"https://openalex.org/keywords/multimodal-learning","display_name":"multimodal learning","score":0.7730000019073486},{"id":"https://openalex.org/keywords/gradient-imbalance","display_name":"gradient imbalance","score":0.7480000257492065},{"id":"https://openalex.org/keywords/plug-in-modules","display_name":"plug-in modules","score":0.6470000147819519},{"id":"https://openalex.org/keywords/speech-emotion-recognition","display_name":"speech emotion recognition","score":0.6510000228881836},{"id":"https://openalex.org/keywords/facial-expressions","display_name":"facial expressions","score":0.6830000281333923},{"id":"https://openalex.org/keywords/tone-of-voice","display_name":"tone of voice","score":0.6610000133514404}],"concepts":[{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.8395137190818787},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.7477478384971619},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6688441038131714},{"id":"https://openalex.org/C2775852435","wikidata":"https://www.wikidata.org/wiki/Q258403","display_name":"Utterance","level":2,"score":0.5889685153961182},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5091971755027771},{"id":"https://openalex.org/C542774811","wikidata":"https://www.wikidata.org/wiki/Q10880526","display_name":"Prosody","level":2,"score":0.4922579824924469},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.47375279664993286},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.4703172743320465},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.453920841217041},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3931192457675934},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.14581075310707092},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.12139594554901123},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3581783.3613846","doi":"https://doi.org/10.1145/3581783.3613846","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3581783.3613846","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3613846","source":{"id":"https://openalex.org/S4363608757","display_name":"ACM International Conference on Multimedia (ACM MM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3581783.3613846","doi":"https://doi.org/10.1145/3581783.3613846","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3581783.3613846","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3613846","source":{"id":"https://openalex.org/S4363608757","display_name":"ACM International Conference on Multimedia (ACM MM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Multimedia","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G3901315400","display_name":null,"funder_award_id":"No.:62006142, No.:U1936203, and No.:62236003","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3981135610","display_name":null,"funder_award_id":"62276155","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5630254371","display_name":null,"funder_award_id":"62236003","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6379011700","display_name":null,"funder_award_id":"GXWD20220817144428005","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7231895066","display_name":null,"funder_award_id":"U1936203","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7519037240","display_name":null,"funder_award_id":"2022CXGC020107","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7627795890","display_name":null,"funder_award_id":"62276155","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7696304900","display_name":null,"funder_award_id":"62006142","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320324777","display_name":"Shandong Jianzhu University","ror":"https://ror.org/01gbfax37"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387969701.pdf","grobid_xml":"https://content.openalex.org/works/W4387969701.grobid-xml"},"referenced_works_count":29,"referenced_works":["https://openalex.org/W2146334809","https://openalex.org/W2572107491","https://openalex.org/W2771951981","https://openalex.org/W2805662932","https://openalex.org/W2891359673","https://openalex.org/W2895615590","https://openalex.org/W2897298014","https://openalex.org/W2943963359","https://openalex.org/W2952228917","https://openalex.org/W2956959730","https://openalex.org/W2964300796","https://openalex.org/W2978360381","https://openalex.org/W3035333188","https://openalex.org/W3092695663","https://openalex.org/W3099056802","https://openalex.org/W3155230099","https://openalex.org/W3173396651","https://openalex.org/W3196591432","https://openalex.org/W3202778561","https://openalex.org/W4221154966","https://openalex.org/W4281701185","https://openalex.org/W4285604359","https://openalex.org/W4304092645","https://openalex.org/W4312583258","https://openalex.org/W4312639100","https://openalex.org/W4367280365","https://openalex.org/W4372260352","https://openalex.org/W4383337770","https://openalex.org/W4384655960"],"related_works":["https://openalex.org/W4417029929","https://openalex.org/W4403792436","https://openalex.org/W7164380828","https://openalex.org/W7162115272","https://openalex.org/W7162428179","https://openalex.org/W4416034187","https://openalex.org/W4416018368","https://openalex.org/W3163470448","https://openalex.org/W7155096087","https://openalex.org/W7166846269"],"abstract_inverted_index":{"Emotion":[0],"recognition":[1],"in":[2,17],"conversation":[3],"aims":[4],"to":[5,43,91],"identify":[6],"the":[7,47,64,93,104],"emotions":[8,23],"underlying":[9],"each":[10],"utterance,":[11],"and":[12,33,110],"it":[13],"has":[14],"great":[15],"potential":[16],"various":[18],"domains.":[19],"Human":[20],"perception":[21],"of":[22,49,74,95,106],"relies":[24],"on":[25],"multiple":[26],"modalities,":[27],"such":[28],"as":[29],"language,":[30],"vocal":[31],"tonality,":[32],"facial":[34],"expressions.":[35],"While":[36],"many":[37],"studies":[38],"have":[39],"incorporated":[40],"multimodal":[41,50],"information":[42],"enhance":[44],"emotion":[45],"recognition,":[46],"performance":[48,105],"models":[51,109],"often":[52],"plateaus":[53],"when":[54],"additional":[55],"modalities":[56],"are":[57],"added.":[58],"We":[59],"demonstrate":[60],"through":[61],"experiments":[62],"that":[63,100],"main":[65],"reason":[66],"for":[67],"this":[68,80],"plateau":[69],"is":[70],"an":[71],"imbalanced":[72],"assignment":[73],"gradients":[75,94],"across":[76],"modalities.":[77,96],"To":[78],"address":[79],"issue,":[81],"we":[82],"propose":[83],"fine-grained":[84],"adaptive":[85],"gradient":[86],"modulation,":[87],"a":[88],"plug-in":[89,113],"approach":[90],"rebalance":[92],"Experimental":[97],"results":[98],"show":[99],"our":[101],"method":[102],"improves":[103],"all":[107],"baseline":[108],"outperforms":[111],"existing":[112],"methods.":[114]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":3}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2023-10-28T00:00:00"}