{"id":"https://openalex.org/W4229065466","doi":"https://doi.org/10.1109/tai.2022.3201809","title":"M2R2: Missing-Modality Robust Emotion Recognition Framework With Iterative Data Augmentation","display_name":"M2R2: Missing-Modality Robust Emotion Recognition Framework With Iterative Data Augmentation","publication_year":2022,"publication_date":"2022-08-26","ids":{"openalex":"https://openalex.org/W4229065466","doi":"https://doi.org/10.1109/tai.2022.3201809"},"language":"en","primary_location":{"id":"doi:10.1109/tai.2022.3201809","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tai.2022.3201809","pdf_url":null,"source":{"id":"https://openalex.org/S4210169448","display_name":"IEEE Transactions on Artificial Intelligence","issn_l":"2691-4581","issn":["2691-4581"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Artificial Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064441924","display_name":"Ning Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I4391768273","display_name":"State Key Laboratory of Electrical Insulation and Power Equipment","ror":"https://ror.org/03kd9rr37","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391768273","https://openalex.org/I87445476"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Ning Wang","raw_affiliation_strings":["Shaanxi Key Laboratory of Smart Grid, the State Key Laboratory of Electrical Insulation and Power Equipment, School of Electrical Engineering, Xi'an Jiaotong University, Xi'an, China"],"affiliations":[{"raw_affiliation_string":"Shaanxi Key Laboratory of Smart Grid, the State Key Laboratory of Electrical Insulation and Power Equipment, School of Electrical Engineering, Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476","https://openalex.org/I4391768273"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084856540","display_name":"Hui Cao","orcid":"https://orcid.org/0000-0003-3301-2713"},"institutions":[{"id":"https://openalex.org/I4391768273","display_name":"State Key Laboratory of Electrical Insulation and Power Equipment","ror":"https://ror.org/03kd9rr37","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391768273","https://openalex.org/I87445476"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hui Cao","raw_affiliation_strings":["Shaanxi Key Laboratory of Smart Grid, the State Key Laboratory of Electrical Insulation and Power Equipment, School of Electrical Engineering, Xi'an Jiaotong University, Xi'an, China"],"affiliations":[{"raw_affiliation_string":"Shaanxi Key Laboratory of Smart Grid, the State Key Laboratory of Electrical Insulation and Power Equipment, School of Electrical Engineering, Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476","https://openalex.org/I4391768273"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056791242","display_name":"Jun Zhao","orcid":"https://orcid.org/0000-0002-6702-0319"},"institutions":[{"id":"https://openalex.org/I4391768273","display_name":"State Key Laboratory of Electrical Insulation and Power Equipment","ror":"https://ror.org/03kd9rr37","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391768273","https://openalex.org/I87445476"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Zhao","raw_affiliation_strings":["Shaanxi Key Laboratory of Smart Grid, the State Key Laboratory of Electrical Insulation and Power Equipment, School of Electrical Engineering, Xi'an Jiaotong University, Xi'an, China"],"affiliations":[{"raw_affiliation_string":"Shaanxi Key Laboratory of Smart Grid, the State Key Laboratory of Electrical Insulation and Power Equipment, School of Electrical Engineering, Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476","https://openalex.org/I4391768273"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101815063","display_name":"Ruilin Chen","orcid":"https://orcid.org/0000-0002-8715-8286"},"institutions":[{"id":"https://openalex.org/I4391768273","display_name":"State Key Laboratory of Electrical Insulation and Power Equipment","ror":"https://ror.org/03kd9rr37","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391768273","https://openalex.org/I87445476"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruilin Chen","raw_affiliation_strings":["Shaanxi Key Laboratory of Smart Grid, the State Key Laboratory of Electrical Insulation and Power Equipment, School of Electrical Engineering, Xi'an Jiaotong University, Xi'an, China"],"affiliations":[{"raw_affiliation_string":"Shaanxi Key Laboratory of Smart Grid, the State Key Laboratory of Electrical Insulation and Power Equipment, School of Electrical Engineering, Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476","https://openalex.org/I4391768273"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101543688","display_name":"Dapeng Yan","orcid":"https://orcid.org/0000-0003-2559-9841"},"institutions":[{"id":"https://openalex.org/I4391768273","display_name":"State Key Laboratory of Electrical Insulation and Power Equipment","ror":"https://ror.org/03kd9rr37","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391768273","https://openalex.org/I87445476"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dapeng Yan","raw_affiliation_strings":["Shaanxi Key Laboratory of Smart Grid, the State Key Laboratory of Electrical Insulation and Power Equipment, School of Electrical Engineering, Xi'an Jiaotong University, Xi'an, China"],"affiliations":[{"raw_affiliation_string":"Shaanxi Key Laboratory of Smart Grid, the State Key Laboratory of Electrical Insulation and Power Equipment, School of Electrical Engineering, Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476","https://openalex.org/I4391768273"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100662644","display_name":"Jie Zhang","orcid":"https://orcid.org/0000-0003-2579-2537"},"institutions":[{"id":"https://openalex.org/I9916479","display_name":"Air Force Medical University","ror":"https://ror.org/00ms48f15","country_code":"CN","type":"education","lineage":["https://openalex.org/I9916479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jie Zhang","raw_affiliation_strings":["Department of Radiation medicine, Preventive Medicine School, the Fourth Military Medical University, Xi'an, China"],"affiliations":[{"raw_affiliation_string":"Department of Radiation medicine, Preventive Medicine School, the Fourth Military Medical University, Xi'an, China","institution_ids":["https://openalex.org/I9916479"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5064441924"],"corresponding_institution_ids":["https://openalex.org/I4391768273","https://openalex.org/I87445476"],"apc_list":null,"apc_paid":null,"fwci":3.6509,"has_fulltext":false,"cited_by_count":20,"citation_normalized_percentile":{"value":0.93058713,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"4","issue":"5","first_page":"1305","last_page":"1316"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9761999845504761,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/utterance","display_name":"Utterance","score":0.7398580312728882},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7381813526153564},{"id":"https://openalex.org/keywords/missing-data","display_name":"Missing data","score":0.6508327722549438},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.6075794696807861},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5890509486198425},{"id":"https://openalex.org/keywords/conversation","display_name":"Conversation","score":0.48553845286369324},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.4759901762008667},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.46536779403686523},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4569634795188904},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4559290409088135},{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"Emotion recognition","score":0.41602885723114014},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3980659246444702},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.1789361536502838},{"id":"https://openalex.org/keywords/communication","display_name":"Communication","score":0.10410085320472717}],"concepts":[{"id":"https://openalex.org/C2775852435","wikidata":"https://www.wikidata.org/wiki/Q258403","display_name":"Utterance","level":2,"score":0.7398580312728882},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7381813526153564},{"id":"https://openalex.org/C9357733","wikidata":"https://www.wikidata.org/wiki/Q6878417","display_name":"Missing data","level":2,"score":0.6508327722549438},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.6075794696807861},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5890509486198425},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.48553845286369324},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4759901762008667},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.46536779403686523},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4569634795188904},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4559290409088135},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.41602885723114014},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3980659246444702},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.1789361536502838},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.10410085320472717},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tai.2022.3201809","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tai.2022.3201809","pdf_url":null,"source":{"id":"https://openalex.org/S4210169448","display_name":"IEEE Transactions on Artificial Intelligence","issn_l":"2691-4581","issn":["2691-4581"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Artificial Intelligence","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.550000011920929,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G3197634902","display_name":null,"funder_award_id":"EIPE21310","funder_id":"https://openalex.org/F4320327040","funder_display_name":"State Key Laboratory of Electrical Insulation and Power Equipment"}],"funders":[{"id":"https://openalex.org/F4320327040","display_name":"State Key Laboratory of Electrical Insulation and Power Equipment","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":51,"referenced_works":["https://openalex.org/W1539625571","https://openalex.org/W1832693441","https://openalex.org/W1924770834","https://openalex.org/W2058787788","https://openalex.org/W2085662862","https://openalex.org/W2109624827","https://openalex.org/W2131774270","https://openalex.org/W2146334809","https://openalex.org/W2181159407","https://openalex.org/W2301358467","https://openalex.org/W2740550900","https://openalex.org/W2764100506","https://openalex.org/W2786411768","https://openalex.org/W2805662932","https://openalex.org/W2887761937","https://openalex.org/W2891359673","https://openalex.org/W2904300840","https://openalex.org/W2906130987","https://openalex.org/W2952667353","https://openalex.org/W2953739332","https://openalex.org/W2963686995","https://openalex.org/W2963873807","https://openalex.org/W2964300796","https://openalex.org/W2965453734","https://openalex.org/W2970431814","https://openalex.org/W3093434718","https://openalex.org/W3098556456","https://openalex.org/W3098802639","https://openalex.org/W3100993965","https://openalex.org/W3112068135","https://openalex.org/W3114214226","https://openalex.org/W3125992765","https://openalex.org/W3156576211","https://openalex.org/W3163470448","https://openalex.org/W3163957165","https://openalex.org/W3169978599","https://openalex.org/W3175825020","https://openalex.org/W3176404283","https://openalex.org/W3195943930","https://openalex.org/W3196059712","https://openalex.org/W3202775307","https://openalex.org/W3207810231","https://openalex.org/W4214876032","https://openalex.org/W4220873374","https://openalex.org/W4230277160","https://openalex.org/W6631190155","https://openalex.org/W6640212811","https://openalex.org/W6735913928","https://openalex.org/W6739901393","https://openalex.org/W6749927861","https://openalex.org/W6762833281"],"related_works":["https://openalex.org/W1968552888","https://openalex.org/W1527532029","https://openalex.org/W2374116601","https://openalex.org/W3093134843","https://openalex.org/W2529301793","https://openalex.org/W2378167147","https://openalex.org/W3210777354","https://openalex.org/W2772323916","https://openalex.org/W3133700904","https://openalex.org/W3191326035"],"abstract_inverted_index":{"This":[0],"article":[1],"deals":[2],"with":[3,9,54,87],"the":[4,23,78,105,110,126,145],"utterance-level":[5],"modalities":[6],"missing":[7,37],"problem":[8,115],"uncertain":[10],"patterns":[11],"on":[12,98,156],"emotion":[13,46,51,154],"recognition":[14,47,52,155],"in":[15],"conversation":[16],"(ERC)":[17],"task.":[18],"Present":[19],"models":[20],"generally":[21],"predict":[22],"speaker's":[24],"emotions":[25],"by":[26,35,58,125],"its":[27],"current":[28],"utterance":[29],"and":[30,81,90,100,141],"context,":[31],"which":[32,49,75],"is":[33,70,93,116],"degraded":[34],"modality":[36],"considerably.":[38],"Our":[39],"work":[40],"proposes":[41],"a":[42,63],"framework":[43],"missing-modality":[44],"robust":[45],"(M2R2),":[48],"trains":[50],"model":[53],"iterative":[55],"data":[56],"augmentation":[57],"learned":[59],"common":[60,111],"representation.":[61],"First,":[62],"network":[64,68],"called":[65],"party":[66],"attentive":[67],"(PANet)":[69],"designed":[71],"to":[72,95,132,136],"classify":[73],"emotions,":[74],"tracks":[76],"all":[77],"speakers'":[79],"states":[80],"context.":[82],"Attention":[83],"mechanism":[84],"between":[85],"speaker":[86],"other":[88],"participants":[89],"dialogue":[91],"topic":[92],"used":[94,130],"decentralize":[96],"dependence":[97],"multitime":[99],"multiparty":[101],"utterances":[102],"instead":[103],"of":[104,147],"possible":[106],"incomplete":[107],"one.":[108],"Moreover,":[109],"representation":[112],"learning":[113],"(CRL)":[114],"defined":[117],"for":[118,152],"modality-missing":[119,153],"problem.":[120],"Data":[121],"imputation":[122],"methods":[123,149],"improved":[124],"adversarial":[127],"strategy":[128],"are":[129],"here":[131],"construct":[133],"extra":[134],"features":[135],"augment":[137],"data.":[138],"Extensive":[139],"experiments":[140],"case":[142],"studies":[143],"validate":[144],"effectiveness":[146],"our":[148],"over":[150],"baselines":[151],"two":[157],"different":[158],"datasets.":[159]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":7}],"updated_date":"2026-04-16T08:26:57.006410","created_date":"2025-10-10T00:00:00"}