{"id":"https://openalex.org/W3174977508","doi":"https://doi.org/10.1109/access.2021.3092735","title":"Multimodal Emotion Recognition Fusion Analysis Adapting BERT With Heterogeneous Feature Unification","display_name":"Multimodal Emotion Recognition Fusion Analysis Adapting BERT With Heterogeneous Feature Unification","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3174977508","doi":"https://doi.org/10.1109/access.2021.3092735","mag":"3174977508"},"language":"en","primary_location":{"id":"doi:10.1109/access.2021.3092735","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3092735","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09466122.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09466122.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100444551","display_name":"Sanghyun Lee","orcid":"https://orcid.org/0000-0002-1107-0122"},"institutions":[{"id":"https://openalex.org/I197347611","display_name":"Korea University","ror":"https://ror.org/047dqcg40","country_code":"KR","type":"education","lineage":["https://openalex.org/I197347611"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Sanghyun Lee","raw_affiliation_strings":["School of Electrical Engineering, Korea University, Seoul, South Korea"],"raw_orcid":"https://orcid.org/0000-0002-1107-0122","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering, Korea University, Seoul, South Korea","institution_ids":["https://openalex.org/I197347611"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102000970","display_name":"David K. Han","orcid":"https://orcid.org/0000-0001-5055-5408"},"institutions":[{"id":"https://openalex.org/I72816309","display_name":"Drexel University","ror":"https://ror.org/04bdffz58","country_code":"US","type":"education","lineage":["https://openalex.org/I72816309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David K. Han","raw_affiliation_strings":["Drexel University, Philadelphia, PA, USA"],"raw_orcid":"https://orcid.org/0000-0001-5055-5408","affiliations":[{"raw_affiliation_string":"Drexel University, Philadelphia, PA, USA","institution_ids":["https://openalex.org/I72816309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5026204977","display_name":"Hanseok Ko","orcid":"https://orcid.org/0000-0002-8744-4514"},"institutions":[{"id":"https://openalex.org/I197347611","display_name":"Korea University","ror":"https://ror.org/047dqcg40","country_code":"KR","type":"education","lineage":["https://openalex.org/I197347611"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Hanseok Ko","raw_affiliation_strings":["School of Electrical Engineering, Korea University, Seoul, South Korea"],"raw_orcid":"https://orcid.org/0000-0002-8744-4514","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering, Korea University, Seoul, South Korea","institution_ids":["https://openalex.org/I197347611"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":8.3674,"has_fulltext":true,"cited_by_count":88,"citation_normalized_percentile":{"value":0.98312915,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"9","issue":null,"first_page":"94557","last_page":"94572"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9836999773979187,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7219827175140381},{"id":"https://openalex.org/keywords/unification","display_name":"Unification","score":0.7159911394119263},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.5488921403884888},{"id":"https://openalex.org/keywords/fusion","display_name":"Fusion","score":0.5196405649185181},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.42513856291770935},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3638994097709656},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.08542299270629883}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7219827175140381},{"id":"https://openalex.org/C96146094","wikidata":"https://www.wikidata.org/wiki/Q609057","display_name":"Unification","level":2,"score":0.7159911394119263},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.5488921403884888},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.5196405649185181},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42513856291770935},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3638994097709656},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.08542299270629883},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2021.3092735","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3092735","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09466122.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:641e8f74bdcd47808eda374665dabd57","is_oa":true,"landing_page_url":"https://doaj.org/article/641e8f74bdcd47808eda374665dabd57","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 9, Pp 94557-94572 (2021)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2021.3092735","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3092735","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09466122.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6899999976158142,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G33248145","display_name":null,"funder_award_id":"10073162","funder_id":"https://openalex.org/F4320334879","funder_display_name":"Korea Evaluation Institute of Industrial Technology"},{"id":"https://openalex.org/G4346960606","display_name":null,"funder_award_id":"10073162","funder_id":"https://openalex.org/F4320321681","funder_display_name":"Ministry of Trade, Industry and Energy"}],"funders":[{"id":"https://openalex.org/F4320321681","display_name":"Ministry of Trade, Industry and Energy","ror":"https://ror.org/008nkqk13"},{"id":"https://openalex.org/F4320334879","display_name":"Korea Evaluation Institute of Industrial Technology","ror":"https://ror.org/03z9cwa38"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3174977508.pdf","grobid_xml":"https://content.openalex.org/works/W3174977508.grobid-xml"},"referenced_works_count":119,"referenced_works":["https://openalex.org/W147964346","https://openalex.org/W1522301498","https://openalex.org/W1546411676","https://openalex.org/W1549016832","https://openalex.org/W1566289585","https://openalex.org/W1628791547","https://openalex.org/W1679826675","https://openalex.org/W1686810756","https://openalex.org/W2081835714","https://openalex.org/W2090777335","https://openalex.org/W2101965618","https://openalex.org/W2110278938","https://openalex.org/W2112555538","https://openalex.org/W2146334809","https://openalex.org/W2153579005","https://openalex.org/W2157052295","https://openalex.org/W2184188583","https://openalex.org/W2194775991","https://openalex.org/W2217426128","https://openalex.org/W2250539671","https://openalex.org/W2295001676","https://openalex.org/W2325939864","https://openalex.org/W2345305417","https://openalex.org/W2395639500","https://openalex.org/W2399733683","https://openalex.org/W2465534249","https://openalex.org/W2525778437","https://openalex.org/W2526050071","https://openalex.org/W2546919788","https://openalex.org/W2556418146","https://openalex.org/W2593116425","https://openalex.org/W2600389231","https://openalex.org/W2605987575","https://openalex.org/W2619383789","https://openalex.org/W2740550900","https://openalex.org/W2741447225","https://openalex.org/W2742274851","https://openalex.org/W2753840835","https://openalex.org/W2786779322","https://openalex.org/W2787581402","https://openalex.org/W2788776247","https://openalex.org/W2798330760","https://openalex.org/W2808359495","https://openalex.org/W2883409523","https://openalex.org/W2886193235","https://openalex.org/W2889286050","https://openalex.org/W2889976648","https://openalex.org/W2893062474","https://openalex.org/W2896094573","https://openalex.org/W2896457183","https://openalex.org/W2899197626","https://openalex.org/W2905551935","https://openalex.org/W2931598392","https://openalex.org/W2936173226","https://openalex.org/W2938404524","https://openalex.org/W2939362715","https://openalex.org/W2946073932","https://openalex.org/W2946165673","https://openalex.org/W2962718314","https://openalex.org/W2962756039","https://openalex.org/W2962931510","https://openalex.org/W2963083845","https://openalex.org/W2963128932","https://openalex.org/W2963341956","https://openalex.org/W2963403868","https://openalex.org/W2963623198","https://openalex.org/W2963685106","https://openalex.org/W2963712289","https://openalex.org/W2963812294","https://openalex.org/W2963871344","https://openalex.org/W2963902314","https://openalex.org/W2964010806","https://openalex.org/W2964051877","https://openalex.org/W2964121744","https://openalex.org/W2964216663","https://openalex.org/W2964230360","https://openalex.org/W2964260444","https://openalex.org/W2964346351","https://openalex.org/W2965453734","https://openalex.org/W2971250720","https://openalex.org/W2973181312","https://openalex.org/W2981171802","https://openalex.org/W2985882473","https://openalex.org/W3007708573","https://openalex.org/W3015964358","https://openalex.org/W3016153892","https://openalex.org/W3034897750","https://openalex.org/W3037572520","https://openalex.org/W3107569919","https://openalex.org/W3205498744","https://openalex.org/W4293665662","https://openalex.org/W4294170691","https://openalex.org/W4297789825","https://openalex.org/W4385245566","https://openalex.org/W6605980550","https://openalex.org/W6631190155","https://openalex.org/W6632670727","https://openalex.org/W6632825141","https://openalex.org/W6637373629","https://openalex.org/W6670991099","https://openalex.org/W6682691769","https://openalex.org/W6686207219","https://openalex.org/W6697498398","https://openalex.org/W6711957290","https://openalex.org/W6719667659","https://openalex.org/W6729831399","https://openalex.org/W6736149460","https://openalex.org/W6739901393","https://openalex.org/W6742023488","https://openalex.org/W6748363073","https://openalex.org/W6748551036","https://openalex.org/W6750417760","https://openalex.org/W6750651883","https://openalex.org/W6752724743","https://openalex.org/W6754426546","https://openalex.org/W6755207826","https://openalex.org/W6762073003","https://openalex.org/W6776048437","https://openalex.org/W6780226713"],"related_works":["https://openalex.org/W2367630557","https://openalex.org/W101666983","https://openalex.org/W2183792531","https://openalex.org/W2595286499","https://openalex.org/W2794344379","https://openalex.org/W2489197555","https://openalex.org/W2471908688","https://openalex.org/W1845283042","https://openalex.org/W2033914206","https://openalex.org/W2042327336"],"abstract_inverted_index":{"Human":[0],"communication":[1,18],"includes":[2],"rich":[3],"emotional":[4,27,32,40],"content,":[5],"thus":[6],"the":[7,25,103,125,130,134,141,160,167,198,214,224,251],"development":[8],"of":[9,24,29,45,87,133,174],"multimodal":[10,97,156,207],"emotion":[11,73,98,107],"recognition":[12,33,74,99,108,225],"plays":[13],"an":[14],"important":[15],"role":[16],"in":[17,38,65],"between":[19],"humans":[20],"and":[21,52,55,119,136,148,176,203,216,254],"computers.":[22],"Because":[23],"complex":[26],"characteristics":[28],"a":[30,35,43,62,95,180,185,220],"speaker,":[31],"remains":[34],"challenge,":[36],"particularly":[37,59],"capturing":[39],"cues":[41,57],"across":[42],"variety":[44],"modalities,":[46],"such":[47],"as":[48],"speech,":[49],"facial":[50],"expressions,":[51],"language.":[53],"Audio":[54],"visual":[56,120,137,177,217],"are":[58,153],"vital":[60],"for":[61,106,191,206,235],"human":[63],"observer":[64],"understanding":[66],"emotions.":[67],"However,":[68],"most":[69],"previous":[70],"work":[71],"on":[72,79,116,239,250],"has":[75],"been":[76],"based":[77,115,155,238],"solely":[78],"linguistic":[80],"information,":[81],"which":[82,152],"can":[83],"overlook":[84],"various":[85],"forms":[86],"nonverbal":[88],"information.":[89],"In":[90,193],"this":[91],"paper,":[92],"we":[93,123,196,246],"present":[94],"new":[96],"approach":[100],"that":[101,213,228,245],"improves":[102],"BERT":[104,126,187],"model":[105,127,188],"by":[109],"combining":[110,184],"it":[111],"with":[112,189],"heterogeneous":[113,131],"features":[114,132,178],"language,":[117],"audio,":[118],"modalities.":[121,138],"Specifically,":[122],"improve":[124],"due":[128],"to":[129,170,223],"audio":[135,175,215],"We":[139,165],"introduce":[140],"Self-Multi-Attention":[142],"Fusion":[143,150],"module,":[144,147,151],"Multi-Attention":[145],"fusion":[146,157],"Video":[149],"attention":[154],"mechanisms":[158],"using":[159],"recently":[161],"proposed":[162],"transformer":[163],"architecture.":[164],"explore":[166],"optimal":[168],"ways":[169],"combine":[171],"fine-grained":[172],"representations":[173],"into":[179],"common":[181],"embedding":[182],"while":[183],"pre-trained":[186],"modalities":[190,230],"fine-tuning.":[192],"our":[194],"experiment,":[195],"evaluate":[197],"commonly":[199],"used":[200],"CMU-MOSI,":[201,252],"CMU-MOSEI,":[202,253],"IEMOCAP":[204,255],"datasets":[205],"sentiment":[208,236],"analysis.":[209],"Ablation":[210],"analysis":[211,237],"indicates":[212],"components":[218],"make":[219],"significant":[221],"contribution":[222],"results,":[226],"suggesting":[227],"these":[229],"contain":[231],"highly":[232],"complementary":[233],"information":[234],"video":[240],"input.":[241],"Our":[242],"method":[243],"shows":[244],"achieve":[247],"state-of-the-art":[248],"performance":[249],"dataset.":[256]},"counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":25},{"year":2024,"cited_by_count":25},{"year":2023,"cited_by_count":20},{"year":2022,"cited_by_count":5}],"updated_date":"2026-07-17T09:13:05.818461","created_date":"2025-10-10T00:00:00"}