{"id":"https://openalex.org/W4321022102","doi":"https://doi.org/10.1109/access.2023.3244390","title":"Multi-Label Multimodal Emotion Recognition With Transformer-Based Fusion and Emotion-Level Representation Learning","display_name":"Multi-Label Multimodal Emotion Recognition With Transformer-Based Fusion and Emotion-Level Representation Learning","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4321022102","doi":"https://doi.org/10.1109/access.2023.3244390"},"language":"en","primary_location":{"id":"doi:10.1109/access.2023.3244390","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3244390","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/10005208/10042438.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/10005208/10042438.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074608299","display_name":"Hoai-Duy Le","orcid":null},"institutions":[{"id":"https://openalex.org/I111277659","display_name":"Chonnam National University","ror":"https://ror.org/05kzjxq56","country_code":"KR","type":"education","lineage":["https://openalex.org/I111277659"]}],"countries":["KR"],"is_corresponding":true,"raw_author_name":"Hoai-Duy Le","raw_affiliation_strings":["Department of Artificial Intelligence Convergence, Chonnam National University, Gwangju, South Korea"],"affiliations":[{"raw_affiliation_string":"Department of Artificial Intelligence Convergence, Chonnam National University, Gwangju, South Korea","institution_ids":["https://openalex.org/I111277659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070936425","display_name":"Guee-Sang Lee","orcid":"https://orcid.org/0000-0002-8756-1382"},"institutions":[{"id":"https://openalex.org/I111277659","display_name":"Chonnam National University","ror":"https://ror.org/05kzjxq56","country_code":"KR","type":"education","lineage":["https://openalex.org/I111277659"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Guee-Sang Lee","raw_affiliation_strings":["Department of Artificial Intelligence Convergence, Chonnam National University, Gwangju, South Korea"],"affiliations":[{"raw_affiliation_string":"Department of Artificial Intelligence Convergence, Chonnam National University, Gwangju, South Korea","institution_ids":["https://openalex.org/I111277659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100605822","display_name":"Soo-Hyung Kim","orcid":"https://orcid.org/0000-0003-3575-5035"},"institutions":[{"id":"https://openalex.org/I111277659","display_name":"Chonnam National University","ror":"https://ror.org/05kzjxq56","country_code":"KR","type":"education","lineage":["https://openalex.org/I111277659"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Soo-Hyung Kim","raw_affiliation_strings":["Department of Artificial Intelligence Convergence, Chonnam National University, Gwangju, South Korea"],"affiliations":[{"raw_affiliation_string":"Department of Artificial Intelligence Convergence, Chonnam National University, Gwangju, South Korea","institution_ids":["https://openalex.org/I111277659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100656391","display_name":"Seungwon Kim","orcid":"https://orcid.org/0000-0003-4221-3058"},"institutions":[{"id":"https://openalex.org/I111277659","display_name":"Chonnam National University","ror":"https://ror.org/05kzjxq56","country_code":"KR","type":"education","lineage":["https://openalex.org/I111277659"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Seungwon Kim","raw_affiliation_strings":["Department of Artificial Intelligence Convergence, Chonnam National University, Gwangju, South Korea"],"affiliations":[{"raw_affiliation_string":"Department of Artificial Intelligence Convergence, Chonnam National University, Gwangju, South Korea","institution_ids":["https://openalex.org/I111277659"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087619194","display_name":"Hyung-Jeong Yang","orcid":"https://orcid.org/0000-0003-3024-5060"},"institutions":[{"id":"https://openalex.org/I111277659","display_name":"Chonnam National University","ror":"https://ror.org/05kzjxq56","country_code":"KR","type":"education","lineage":["https://openalex.org/I111277659"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Hyung-Jeong Yang","raw_affiliation_strings":["Department of Artificial Intelligence Convergence, Chonnam National University, Gwangju, South Korea"],"affiliations":[{"raw_affiliation_string":"Department of Artificial Intelligence Convergence, Chonnam National University, Gwangju, South Korea","institution_ids":["https://openalex.org/I111277659"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5074608299"],"corresponding_institution_ids":["https://openalex.org/I111277659"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":12.902,"has_fulltext":true,"cited_by_count":74,"citation_normalized_percentile":{"value":0.99117959,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"11","issue":null,"first_page":"14742","last_page":"14751"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8420008420944214},{"id":"https://openalex.org/keywords/feature-learning","display_name":"Feature learning","score":0.6461772322654724},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5969868898391724},{"id":"https://openalex.org/keywords/multimodal-learning","display_name":"Multimodal learning","score":0.594045877456665},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.570540726184845},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.5292263031005859},{"id":"https://openalex.org/keywords/multi-task-learning","display_name":"Multi-task learning","score":0.4783896207809448},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.45810699462890625},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4348374307155609},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.42356887459754944},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.34946900606155396},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.34142744541168213}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8420008420944214},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.6461772322654724},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5969868898391724},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.594045877456665},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.570540726184845},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.5292263031005859},{"id":"https://openalex.org/C28006648","wikidata":"https://www.wikidata.org/wiki/Q6934509","display_name":"Multi-task learning","level":3,"score":0.4783896207809448},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.45810699462890625},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4348374307155609},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.42356887459754944},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.34946900606155396},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.34142744541168213},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2023.3244390","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3244390","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/10005208/10042438.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:8f72a639eb4c49fdb5b2192374b9fdfc","is_oa":true,"landing_page_url":"https://doaj.org/article/8f72a639eb4c49fdb5b2192374b9fdfc","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 11, Pp 14742-14751 (2023)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2023.3244390","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3244390","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/10005208/10042438.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.44999998807907104,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G2792493025","display_name":null,"funder_award_id":"NRF-2020R1A4A1019191","funder_id":"https://openalex.org/F4320328359","funder_display_name":"Ministry of Science and ICT, South Korea"},{"id":"https://openalex.org/G3298389238","display_name":null,"funder_award_id":"NRF-2020R1A4A1019191","funder_id":"https://openalex.org/F4320322030","funder_display_name":"Ministry of Science, ICT and Future Planning"},{"id":"https://openalex.org/G342704958","display_name":null,"funder_award_id":"funded","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"},{"id":"https://openalex.org/G3915194844","display_name":null,"funder_award_id":"NRF-2020R1A4A1019191","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"},{"id":"https://openalex.org/G6285610340","display_name":null,"funder_award_id":"2020R1A4A1019191","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"},{"id":"https://openalex.org/G6416076947","display_name":null,"funder_award_id":"NRF-2020R1A4A1019191","funder_id":"https://openalex.org/F4320320671","funder_display_name":"National Research Foundation"},{"id":"https://openalex.org/G7598120613","display_name":null,"funder_award_id":"NRF-2020R1A4A10","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"},{"id":"https://openalex.org/G7685055460","display_name":null,"funder_award_id":"Grant","funder_id":"https://openalex.org/F4320328359","funder_display_name":"Ministry of Science and ICT, South Korea"},{"id":"https://openalex.org/G865289537","display_name":null,"funder_award_id":"Korea Government (MSIT)","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"},{"id":"https://openalex.org/G982292920","display_name":null,"funder_award_id":"NRF-20","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"}],"funders":[{"id":"https://openalex.org/F4320320671","display_name":"National Research Foundation","ror":"https://ror.org/05s0g1g46"},{"id":"https://openalex.org/F4320322030","display_name":"Ministry of Science, ICT and Future Planning","ror":"https://ror.org/032e49973"},{"id":"https://openalex.org/F4320322120","display_name":"National Research Foundation of Korea","ror":"https://ror.org/013aysd81"},{"id":"https://openalex.org/F4320322724","display_name":"Ministry of Education, India","ror":"https://ror.org/048xjjh50"},{"id":"https://openalex.org/F4320328359","display_name":"Ministry of Science and ICT, South Korea","ror":"https://ror.org/01wpjm123"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4321022102.pdf","grobid_xml":"https://content.openalex.org/works/W4321022102.grobid-xml"},"referenced_works_count":40,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1686810756","https://openalex.org/W1902237438","https://openalex.org/W2146334809","https://openalex.org/W2548844710","https://openalex.org/W2612690371","https://openalex.org/W2769905613","https://openalex.org/W2883409523","https://openalex.org/W2885719698","https://openalex.org/W2963263347","https://openalex.org/W2964010806","https://openalex.org/W2964051877","https://openalex.org/W2964346351","https://openalex.org/W2970231061","https://openalex.org/W2991485489","https://openalex.org/W2997136715","https://openalex.org/W3007282427","https://openalex.org/W3048939150","https://openalex.org/W3087434251","https://openalex.org/W3093051361","https://openalex.org/W3096609285","https://openalex.org/W3099638501","https://openalex.org/W3184087575","https://openalex.org/W3206529771","https://openalex.org/W3206996142","https://openalex.org/W4286593296","https://openalex.org/W4312384316","https://openalex.org/W4312653797","https://openalex.org/W6631190155","https://openalex.org/W6637373629","https://openalex.org/W6726497184","https://openalex.org/W6739901393","https://openalex.org/W6766978945","https://openalex.org/W6768021236","https://openalex.org/W6783497617","https://openalex.org/W6792098614","https://openalex.org/W6793736971","https://openalex.org/W6797613833","https://openalex.org/W6798617289","https://openalex.org/W6801230018"],"related_works":["https://openalex.org/W4389505417","https://openalex.org/W2962931510","https://openalex.org/W4380551887","https://openalex.org/W2237537322","https://openalex.org/W4285159263","https://openalex.org/W2950678851","https://openalex.org/W4301248618","https://openalex.org/W2904518532","https://openalex.org/W4280529741","https://openalex.org/W4293919860"],"abstract_inverted_index":{"Emotion":[0,185],"recognition":[1,15,55],"has":[2,19],"been":[3,71],"an":[4],"active":[5],"research":[6],"area":[7],"for":[8,53,63,105,139,208],"a":[9,89,135,141],"long":[10],"time.":[11],"Recently,":[12],"multimodal":[13,51,78,100,143],"emotion":[14,111,211],"from":[16,42,102,130],"video":[17,27,35,46,110,118,210],"data":[18,47],"grown":[20],"in":[21,45],"importance":[22],"with":[23,154],"the":[24,31,60,106,148,155,161,198],"explosion":[25],"of":[26,33,108],"content":[28],"due":[29],"to":[30,48,96,152,188],"emergence":[32],"short":[34],"social":[36],"media":[37],"platforms.":[38],"Effectively":[39],"incorporating":[40],"information":[41,129],"multiple":[43,132],"modalities":[44,133],"learn":[49],"robust":[50],"representation":[52,82,93,150],"improving":[54],"model":[56,162],"performance":[57],"is":[58],"still":[59],"primary":[61],"challenge":[62],"researchers.":[64],"In":[65],"this":[66],"context,":[67],"transformer":[68,137],"architectures":[69],"have":[70,75],"widely":[72],"used":[73],"and":[74,81,92,98,122,127,159,177,184,205],"significantly":[76],"improved":[77],"deep":[79],"learning":[80,94,140],"learning.":[83],"Inspired":[84],"by":[85],"this,":[86],"we":[87,146],"propose":[88],"transformer-based":[90],"fusion":[91],"method":[95,115,200],"fuse":[97],"enrich":[99],"features":[101],"raw":[103,117],"videos":[104],"task":[107,158],"multi-label":[109,156,209],"recognition.":[112,212],"Specifically,":[113],"our":[114,190],"takes":[116],"frames,":[119],"audio":[120],"signals,":[121],"text":[123],"subtitles":[124],"as":[125],"inputs":[126],"passes":[128],"these":[131],"through":[134],"unified":[136],"architecture":[138],"joint":[142],"representation.":[144],"Moreover,":[145],"use":[147],"label-level":[149],"approach":[151],"deal":[153],"classification":[157],"enhance":[160],"performance.":[163],"We":[164],"conduct":[165],"experiments":[166],"on":[167],"two":[168],"benchmark":[169],"datasets:":[170],"Interactive":[171],"Emotional":[172],"Dyadic":[173],"Motion":[174],"Capture":[175],"(IEMOCAP)":[176],"Carnegie":[178],"Mellon":[179],"University":[180],"Multimodal":[181],"Opinion":[182],"Sentiment":[183],"Intensity":[186],"(CMU-MOSEI)":[187],"evaluate":[189],"proposed":[191,199],"method.":[192],"The":[193],"experimental":[194],"results":[195],"demonstrate":[196],"that":[197],"outperforms":[201],"other":[202],"strong":[203],"baselines":[204],"existing":[206],"approaches":[207]},"counts_by_year":[{"year":2026,"cited_by_count":10},{"year":2025,"cited_by_count":29},{"year":2024,"cited_by_count":28},{"year":2023,"cited_by_count":7}],"updated_date":"2026-03-27T14:29:43.386196","created_date":"2025-10-10T00:00:00"}