{"id":"https://openalex.org/W3159301005","doi":"https://doi.org/10.1109/tcds.2021.3071170","title":"Comparing Recognition Performance and Robustness of Multimodal Deep Learning Models for Multimodal Emotion Recognition","display_name":"Comparing Recognition Performance and Robustness of Multimodal Deep Learning Models for Multimodal Emotion Recognition","publication_year":2022,"publication_date":"2022-06-01","ids":{"openalex":"https://openalex.org/W3159301005","doi":"https://doi.org/10.1109/tcds.2021.3071170","mag":"3159301005"},"language":"en","primary_location":{"id":"doi:10.1109/tcds.2021.3071170","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcds.2021.3071170","pdf_url":null,"source":{"id":"https://openalex.org/S2488537894","display_name":"IEEE Transactions on Cognitive and Developmental Systems","issn_l":"2379-8920","issn":["2379-8920","2379-8939"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cognitive and Developmental Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5079066710","display_name":"Wei Liu","orcid":"https://orcid.org/0000-0002-3840-1980"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I2801556517","display_name":"Ruijin Hospital","ror":"https://ror.org/01hv94n30","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I2801556517"]},{"id":"https://openalex.org/I4210165726","display_name":"Shanghai Municipal Education Commission","ror":"https://ror.org/05tewj457","country_code":"CN","type":"government","lineage":["https://openalex.org/I4210165456","https://openalex.org/I4210165726"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Wei Liu","raw_affiliation_strings":["Department of Computer Science and Engineering, the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Brain Science and Technology Research Center, Center for Brain-Like Computing and Machine Intelligence, Qing Yuan Research Institute, Shanghai Jiao Tong University, Shanghai, China","Center for Brain-Machine Interface and Neuromodulation, Rui-Jin Hospital, Shanghai Jiao Tong University School of Medicine, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Brain Science and Technology Research Center, Center for Brain-Like Computing and Machine Intelligence, Qing Yuan Research Institute, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I4210165726","https://openalex.org/I183067930"]},{"raw_affiliation_string":"Center for Brain-Machine Interface and Neuromodulation, Rui-Jin Hospital, Shanghai Jiao Tong University School of Medicine, Shanghai, China","institution_ids":["https://openalex.org/I2801556517","https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058194829","display_name":"Jielin Qiu","orcid":"https://orcid.org/0000-0002-7384-1324"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jie-Lin Qiu","raw_affiliation_strings":["Computer Science Department, Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Computer Science Department, Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056335002","display_name":"Wei\u2010Long Zheng","orcid":"https://orcid.org/0000-0002-9474-6369"},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wei-Long Zheng","raw_affiliation_strings":["Department of Brain and Cognitive Science, Massachusetts Institute of Technology, Cambridge, MA, USA"],"affiliations":[{"raw_affiliation_string":"Department of Brain and Cognitive Science, Massachusetts Institute of Technology, Cambridge, MA, USA","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040440605","display_name":"Bao\u2010Liang Lu","orcid":"https://orcid.org/0000-0001-8359-0058"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I2801556517","display_name":"Ruijin Hospital","ror":"https://ror.org/01hv94n30","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I2801556517"]},{"id":"https://openalex.org/I4210165726","display_name":"Shanghai Municipal Education Commission","ror":"https://ror.org/05tewj457","country_code":"CN","type":"government","lineage":["https://openalex.org/I4210165456","https://openalex.org/I4210165726"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bao-Liang Lu","raw_affiliation_strings":["Department of Computer Science and Engineering, the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Brain Science and Technology Research Center, Center for Brain-Like Computing and Machine Intelligence, Qing Yuan Research Institute, Shanghai Jiao Tong University, Shanghai, China","Center for Brain-Machine Interface and Neuromodulation, Rui-Jin Hospital, Shanghai Jiao Tong University School of Medicine, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Brain Science and Technology Research Center, Center for Brain-Like Computing and Machine Intelligence, Qing Yuan Research Institute, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I4210165726","https://openalex.org/I183067930"]},{"raw_affiliation_string":"Center for Brain-Machine Interface and Neuromodulation, Rui-Jin Hospital, Shanghai Jiao Tong University School of Medicine, Shanghai, China","institution_ids":["https://openalex.org/I2801556517","https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5079066710"],"corresponding_institution_ids":["https://openalex.org/I183067930","https://openalex.org/I2801556517","https://openalex.org/I4210165726"],"apc_list":null,"apc_paid":null,"fwci":68.7955,"has_fulltext":false,"cited_by_count":389,"citation_normalized_percentile":{"value":0.99972278,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"14","issue":"2","first_page":"715","last_page":"729"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10429","display_name":"EEG and Brain-Computer Interfaces","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.7962862849235535},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7728147506713867},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.7194230556488037},{"id":"https://openalex.org/keywords/discriminative-model","display_name":"Discriminative model","score":0.6268378496170044},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.5926429033279419},{"id":"https://openalex.org/keywords/autoencoder","display_name":"Autoencoder","score":0.5169003009796143},{"id":"https://openalex.org/keywords/data-set","display_name":"Data set","score":0.48699575662612915},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3882797956466675},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.3856794238090515},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.3755224049091339}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.7962862849235535},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7728147506713867},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7194230556488037},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.6268378496170044},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.5926429033279419},{"id":"https://openalex.org/C101738243","wikidata":"https://www.wikidata.org/wiki/Q786435","display_name":"Autoencoder","level":3,"score":0.5169003009796143},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.48699575662612915},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3882797956466675},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.3856794238090515},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3755224049091339},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tcds.2021.3071170","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcds.2021.3071170","pdf_url":null,"source":{"id":"https://openalex.org/S2488537894","display_name":"IEEE Transactions on Cognitive and Developmental Systems","issn_l":"2379-8920","issn":["2379-8920","2379-8939"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cognitive and Developmental Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities","score":0.6499999761581421}],"awards":[{"id":"https://openalex.org/G143590983","display_name":null,"funder_award_id":"61976135","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5385248375","display_name":null,"funder_award_id":"2017YFB1002501","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G6952365848","display_name":null,"funder_award_id":"GDKJXM20185761","funder_id":"https://openalex.org/F4320323192","funder_display_name":"China Southern Power Grid"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320323192","display_name":"China Southern Power Grid","ror":"https://ror.org/03hkh9419"},{"id":"https://openalex.org/F4320327912","display_name":"Higher Education Discipline Innovation Project","ror":null},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":70,"referenced_works":["https://openalex.org/W129606432","https://openalex.org/W605727707","https://openalex.org/W1523385540","https://openalex.org/W1539811621","https://openalex.org/W1939789023","https://openalex.org/W1947251450","https://openalex.org/W1970727126","https://openalex.org/W2002055708","https://openalex.org/W2002565627","https://openalex.org/W2005178092","https://openalex.org/W2008244698","https://openalex.org/W2038610431","https://openalex.org/W2123024445","https://openalex.org/W2139564752","https://openalex.org/W2164368909","https://openalex.org/W2164699598","https://openalex.org/W2169522260","https://openalex.org/W2289165730","https://openalex.org/W2339343773","https://openalex.org/W2407512993","https://openalex.org/W2517194566","https://openalex.org/W2525648609","https://openalex.org/W2547788783","https://openalex.org/W2565944610","https://openalex.org/W2572280252","https://openalex.org/W2584561145","https://openalex.org/W2598673810","https://openalex.org/W2599124244","https://openalex.org/W2604825588","https://openalex.org/W2618843390","https://openalex.org/W2619383789","https://openalex.org/W2745497104","https://openalex.org/W2766825571","https://openalex.org/W2778978785","https://openalex.org/W2786768213","https://openalex.org/W2787581402","https://openalex.org/W2790404832","https://openalex.org/W2792191740","https://openalex.org/W2792267851","https://openalex.org/W2809449684","https://openalex.org/W2900037010","https://openalex.org/W2901809396","https://openalex.org/W2926366943","https://openalex.org/W2945331847","https://openalex.org/W2945585469","https://openalex.org/W2945600262","https://openalex.org/W2950162539","https://openalex.org/W2962905870","https://openalex.org/W2963176022","https://openalex.org/W2964346351","https://openalex.org/W2970972665","https://openalex.org/W2976149155","https://openalex.org/W3002833466","https://openalex.org/W3011145831","https://openalex.org/W3011304975","https://openalex.org/W3012948425","https://openalex.org/W3025868130","https://openalex.org/W3048484056","https://openalex.org/W3089557188","https://openalex.org/W3106275434","https://openalex.org/W3122081138","https://openalex.org/W4230277160","https://openalex.org/W4232478844","https://openalex.org/W6631216910","https://openalex.org/W6678470764","https://openalex.org/W6678846912","https://openalex.org/W6686207219","https://openalex.org/W6713572877","https://openalex.org/W6752051073","https://openalex.org/W6767287485"],"related_works":["https://openalex.org/W2159052453","https://openalex.org/W3013693939","https://openalex.org/W2566616303","https://openalex.org/W3131327266","https://openalex.org/W2752972570","https://openalex.org/W4297051394","https://openalex.org/W2734887215","https://openalex.org/W2803255133","https://openalex.org/W2669956259","https://openalex.org/W4249005693"],"abstract_inverted_index":{"Multimodal":[0],"signals":[1],"are":[2,46,214],"powerful":[3],"for":[4,53,59],"emotion":[5,26],"recognition":[6,19,27,129],"since":[7],"they":[8],"can":[9],"represent":[10],"emotions":[11],"comprehensively.":[12],"In":[13],"this":[14,44],"article,":[15],"we":[16,49,71,89,208],"compare":[17,73],"the":[18,55,74,91,139,146,155,162,173,189,204,211],"performance":[20,75],"and":[21,35,66,79,87,96,101,114,152,166,170,191,199,217],"robustness":[22,92,181],"of":[23,43,76,93,186],"two":[24,51,106],"multimodal":[25,60,84,112],"models:":[28],"1)":[29,48,108,136],"deep":[30,38],"canonical":[31],"correlation":[32],"analysis":[33],"(DCCA)":[34],"2)":[36,70,115,143],"bimodal":[37],"autoencoder":[39],"(BDAE).":[40],"The":[41],"contributions":[42],"article":[45],"threefold:":[47],"propose":[50],"methods":[52],"extending":[54],"original":[56],"DCCA":[57,126,178,201],"model":[58],"fusion:":[61],"a)":[62],"weighted":[63],"sum":[64],"fusion":[65],"b)":[67],"attention-based":[68],"fusion;":[69],"systemically":[72],"DCCA,":[77,94],"BDAE,":[78,95],"traditional":[80,97],"approaches":[81,98],"on":[82,99,131,138,145,154,161,172,203],"five":[83,133],"data":[85,103,134,141,148,157,164,175,193,206],"sets;":[86],"3)":[88,150],"investigate":[90],"SEED-V":[100,163,190,205],"DREAMER":[102,174,192],"sets":[104],"under":[105],"conditions:":[107],"adding":[109,183],"noises":[110,187],"to":[111,188],"features":[113,118,197,213],"replacing":[116],"electroencephalography":[117],"with":[119],"noises.":[120],"Our":[121],"experimental":[122],"results":[123,130],"demonstrate":[124],"that":[125,210],"achieves":[127],"state-of-the-art":[128],"all":[132],"sets:":[135],"94.6%":[137],"SEED":[140],"set;":[142,149,158,165],"87.5%":[144],"SEED-IV":[147],"84.3%":[151],"85.6%":[153],"DEAP":[156],"4)":[159],"85.3%":[160],"5)":[167],"89.0%,":[168],"90.6%,":[169],"90.7%":[171],"set.":[176],"Meanwhile,":[177],"has":[179],"greater":[180],"when":[182],"various":[184],"amounts":[185],"sets.":[194],"By":[195],"visualizing":[196],"before":[198],"after":[200],"transformation":[202],"set,":[207],"find":[209],"transformed":[212],"more":[215],"homogeneous":[216],"discriminative":[218],"across":[219],"emotions.":[220]},"counts_by_year":[{"year":2026,"cited_by_count":27},{"year":2025,"cited_by_count":143},{"year":2024,"cited_by_count":114},{"year":2023,"cited_by_count":75},{"year":2022,"cited_by_count":26},{"year":2021,"cited_by_count":4}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}