{"id":"https://openalex.org/W3032240740","doi":"https://doi.org/10.1109/tip.2020.2996086","title":"Multi-Modal Recurrent Attention Networks for Facial Expression Recognition","display_name":"Multi-Modal Recurrent Attention Networks for Facial Expression Recognition","publication_year":2020,"publication_date":"2020-01-01","ids":{"openalex":"https://openalex.org/W3032240740","doi":"https://doi.org/10.1109/tip.2020.2996086","mag":"3032240740"},"language":"en","primary_location":{"id":"doi:10.1109/tip.2020.2996086","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tip.2020.2996086","pdf_url":null,"source":{"id":"https://openalex.org/S4210173141","display_name":"IEEE Transactions on Image Processing","issn_l":"1057-7149","issn":["1057-7149","1941-0042"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Image Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://infoscience.epfl.ch/record/278821","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100447461","display_name":"Jiyoung Lee","orcid":"https://orcid.org/0000-0003-0840-3317"},"institutions":[{"id":"https://openalex.org/I193775966","display_name":"Yonsei University","ror":"https://ror.org/01wjejq96","country_code":"KR","type":"education","lineage":["https://openalex.org/I193775966"]}],"countries":["KR"],"is_corresponding":true,"raw_author_name":"Jiyoung Lee","raw_affiliation_strings":["School of Electrical and Electronic Engineering, Yonsei University, Seoul, South Korea"],"raw_orcid":"https://orcid.org/0000-0003-0840-3317","affiliations":[{"raw_affiliation_string":"School of Electrical and Electronic Engineering, Yonsei University, Seoul, South Korea","institution_ids":["https://openalex.org/I193775966"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002561350","display_name":"Sunok Kim","orcid":"https://orcid.org/0000-0002-9665-4214"},"institutions":[{"id":"https://openalex.org/I193775966","display_name":"Yonsei University","ror":"https://ror.org/01wjejq96","country_code":"KR","type":"education","lineage":["https://openalex.org/I193775966"]},{"id":"https://openalex.org/I197347611","display_name":"Korea University","ror":"https://ror.org/047dqcg40","country_code":"KR","type":"education","lineage":["https://openalex.org/I197347611"]},{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH","KR"],"is_corresponding":false,"raw_author_name":"Sunok Kim","raw_affiliation_strings":["Department of Computer Science and Engineering, Korea University, Seoul, South Korea","School of Electrical and Electronic Engineering, Yonsei University, Seoul, South Korea","\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne (EPFL), Lausanne, Switzerland"],"raw_orcid":"https://orcid.org/0000-0002-9665-4214","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Korea University, Seoul, South Korea","institution_ids":["https://openalex.org/I197347611"]},{"raw_affiliation_string":"School of Electrical and Electronic Engineering, Yonsei University, Seoul, South Korea","institution_ids":["https://openalex.org/I193775966"]},{"raw_affiliation_string":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne (EPFL), Lausanne, Switzerland","institution_ids":["https://openalex.org/I5124864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085363061","display_name":"Seungryong Kim","orcid":"https://orcid.org/0000-0003-2927-6273"},"institutions":[{"id":"https://openalex.org/I193775966","display_name":"Yonsei University","ror":"https://ror.org/01wjejq96","country_code":"KR","type":"education","lineage":["https://openalex.org/I193775966"]},{"id":"https://openalex.org/I197347611","display_name":"Korea University","ror":"https://ror.org/047dqcg40","country_code":"KR","type":"education","lineage":["https://openalex.org/I197347611"]},{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH","KR"],"is_corresponding":false,"raw_author_name":"Seungryong Kim","raw_affiliation_strings":["Department of Computer Science and Engineering, Korea University, Seoul, South Korea","School of Electrical and Electronic Engineering, Yonsei University, Seoul, South Korea","\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne (EPFL), Lausanne, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Korea University, Seoul, South Korea","institution_ids":["https://openalex.org/I197347611"]},{"raw_affiliation_string":"School of Electrical and Electronic Engineering, Yonsei University, Seoul, South Korea","institution_ids":["https://openalex.org/I193775966"]},{"raw_affiliation_string":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne (EPFL), Lausanne, Switzerland","institution_ids":["https://openalex.org/I5124864"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5073320959","display_name":"Kwanghoon Sohn","orcid":"https://orcid.org/0000-0002-3715-0331"},"institutions":[{"id":"https://openalex.org/I193775966","display_name":"Yonsei University","ror":"https://ror.org/01wjejq96","country_code":"KR","type":"education","lineage":["https://openalex.org/I193775966"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Kwanghoon Sohn","raw_affiliation_strings":["School of Electrical and Electronic Engineering, Yonsei University, Seoul, South Korea"],"raw_orcid":"https://orcid.org/0000-0002-3715-0331","affiliations":[{"raw_affiliation_string":"School of Electrical and Electronic Engineering, Yonsei University, Seoul, South Korea","institution_ids":["https://openalex.org/I193775966"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5100447461"],"corresponding_institution_ids":["https://openalex.org/I193775966"],"apc_list":null,"apc_paid":null,"fwci":9.4875,"has_fulltext":false,"cited_by_count":91,"citation_normalized_percentile":{"value":0.98363867,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"29","issue":null,"first_page":"6977","last_page":"6991"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10057","display_name":"Face and Expression Recognition","score":0.9944000244140625,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11707","display_name":"Gaze Tracking and Assistive Technology","score":0.9883999824523926,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/facial-expression","display_name":"Facial expression","score":0.80201655626297},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7156575322151184},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6964468359947205},{"id":"https://openalex.org/keywords/discriminative-model","display_name":"Discriminative model","score":0.683038055896759},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.5469496846199036},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.5438353419303894},{"id":"https://openalex.org/keywords/expression","display_name":"Expression (computer science)","score":0.5223268866539001},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.5114918947219849},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.49778056144714355},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.43889063596725464},{"id":"https://openalex.org/keywords/facial-recognition-system","display_name":"Facial recognition system","score":0.438593327999115},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.34832078218460083}],"concepts":[{"id":"https://openalex.org/C195704467","wikidata":"https://www.wikidata.org/wiki/Q327968","display_name":"Facial expression","level":2,"score":0.80201655626297},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7156575322151184},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6964468359947205},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.683038055896759},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.5469496846199036},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.5438353419303894},{"id":"https://openalex.org/C90559484","wikidata":"https://www.wikidata.org/wiki/Q778379","display_name":"Expression (computer science)","level":2,"score":0.5223268866539001},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5114918947219849},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.49778056144714355},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.43889063596725464},{"id":"https://openalex.org/C31510193","wikidata":"https://www.wikidata.org/wiki/Q1192553","display_name":"Facial recognition system","level":3,"score":0.438593327999115},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.34832078218460083},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tip.2020.2996086","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tip.2020.2996086","pdf_url":null,"source":{"id":"https://openalex.org/S4210173141","display_name":"IEEE Transactions on Image Processing","issn_l":"1057-7149","issn":["1057-7149","1941-0042"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Image Processing","raw_type":"journal-article"},{"id":"pmh:oai:infoscience.epfl.ch:278821","is_oa":true,"landing_page_url":"http://infoscience.epfl.ch/record/278821","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"WoS","raw_type":"research article"}],"best_oa_location":{"id":"pmh:oai:infoscience.epfl.ch:278821","is_oa":true,"landing_page_url":"http://infoscience.epfl.ch/record/278821","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"WoS","raw_type":"research article"},"sustainable_development_goals":[{"score":0.75,"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities"}],"awards":[{"id":"https://openalex.org/G4830013026","display_name":null,"funder_award_id":"NRF-2018M3E3A1057289","funder_id":"https://openalex.org/F4320322030","funder_display_name":"Ministry of Science, ICT and Future Planning"}],"funders":[{"id":"https://openalex.org/F4320322030","display_name":"Ministry of Science, ICT and Future Planning","ror":"https://ror.org/032e49973"},{"id":"https://openalex.org/F4320322120","display_name":"National Research Foundation of Korea","ror":"https://ror.org/013aysd81"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":102,"referenced_works":["https://openalex.org/W1485009520","https://openalex.org/W1503473358","https://openalex.org/W1522301498","https://openalex.org/W1533861849","https://openalex.org/W1588539311","https://openalex.org/W1686810756","https://openalex.org/W1811254738","https://openalex.org/W1964744287","https://openalex.org/W1964757081","https://openalex.org/W1965696296","https://openalex.org/W1976948919","https://openalex.org/W1981918162","https://openalex.org/W1982709543","https://openalex.org/W1983364832","https://openalex.org/W1992227055","https://openalex.org/W2003653478","https://openalex.org/W2005418748","https://openalex.org/W2008933718","https://openalex.org/W2017626840","https://openalex.org/W2018958905","https://openalex.org/W2019312772","https://openalex.org/W2021913835","https://openalex.org/W2025905516","https://openalex.org/W2045528981","https://openalex.org/W2051297709","https://openalex.org/W2062632672","https://openalex.org/W2072038603","https://openalex.org/W2081835714","https://openalex.org/W2092206588","https://openalex.org/W2096733369","https://openalex.org/W2102570318","https://openalex.org/W2103943262","https://openalex.org/W2106115875","https://openalex.org/W2106947945","https://openalex.org/W2108598243","https://openalex.org/W2113410615","https://openalex.org/W2115252128","https://openalex.org/W2127462305","https://openalex.org/W2128272608","https://openalex.org/W2132555391","https://openalex.org/W2137306662","https://openalex.org/W2139916508","https://openalex.org/W2141399712","https://openalex.org/W2145287260","https://openalex.org/W2154961933","https://openalex.org/W2167557160","https://openalex.org/W2172806452","https://openalex.org/W2194775991","https://openalex.org/W2195207531","https://openalex.org/W2220981600","https://openalex.org/W2277498883","https://openalex.org/W2278113816","https://openalex.org/W2281407413","https://openalex.org/W2295107390","https://openalex.org/W2325939864","https://openalex.org/W2340671224","https://openalex.org/W2346454595","https://openalex.org/W2470957930","https://openalex.org/W2583855388","https://openalex.org/W2587982884","https://openalex.org/W2600389231","https://openalex.org/W2606594151","https://openalex.org/W2739474071","https://openalex.org/W2751214333","https://openalex.org/W2768799425","https://openalex.org/W2791271581","https://openalex.org/W2792925809","https://openalex.org/W2798503473","https://openalex.org/W2798536775","https://openalex.org/W2884585870","https://openalex.org/W2887068828","https://openalex.org/W2889978276","https://openalex.org/W2894871570","https://openalex.org/W2899366898","https://openalex.org/W2910165986","https://openalex.org/W2915606245","https://openalex.org/W2962706528","https://openalex.org/W2963246338","https://openalex.org/W2963252191","https://openalex.org/W2964121744","https://openalex.org/W2970971581","https://openalex.org/W2972850892","https://openalex.org/W3098061148","https://openalex.org/W3099206234","https://openalex.org/W3099268463","https://openalex.org/W3100513545","https://openalex.org/W4236533540","https://openalex.org/W4294557331","https://openalex.org/W4295312788","https://openalex.org/W6630045706","https://openalex.org/W6631190155","https://openalex.org/W6631943919","https://openalex.org/W6637373629","https://openalex.org/W6638742206","https://openalex.org/W6670991099","https://openalex.org/W6677618333","https://openalex.org/W6680591342","https://openalex.org/W6680834391","https://openalex.org/W6684983439","https://openalex.org/W6700903540","https://openalex.org/W6753412334","https://openalex.org/W6766978945"],"related_works":["https://openalex.org/W2965546495","https://openalex.org/W4389116644","https://openalex.org/W2153315159","https://openalex.org/W3103844505","https://openalex.org/W259157601","https://openalex.org/W4205463238","https://openalex.org/W2761785940","https://openalex.org/W2110523656","https://openalex.org/W1482209366","https://openalex.org/W2985118265"],"abstract_inverted_index":{"Recent":[0],"deep":[1],"neural":[2],"networks":[3,92],"based":[4,104],"methods":[5],"have":[6,25],"achieved":[7],"state-of-the-art":[8,172],"performance":[9],"on":[10,28,105,125,179],"various":[11],"facial":[12,22,51,102,137,144,176],"expression":[13,23,103,138,145,177],"recognition":[14,24,146,178],"tasks.":[15],"Despite":[16],"such":[17],"progress,":[18],"previous":[19],"researches":[20],"for":[21,119,135],"mainly":[26],"focused":[27],"analyzing":[29],"color":[30,120,180],"recording":[31,78,155,181,191],"videos":[32,79,156],"only.":[33],"However,":[34],"the":[35,101,111,171],"complex":[36],"emotions":[37],"expressed":[38],"by":[39,57],"people":[40],"with":[41,157],"different":[42,46],"skin":[43],"colors":[44],"under":[45],"lighting":[47],"conditions":[48],"through":[49],"dynamic":[50],"expressions":[52],"can":[53,169],"be":[54],"fully":[55],"understandable":[56],"integrating":[58],"information":[59],"from":[60],"multi-modal":[61,84,89,136,142,190],"videos.":[62],"We":[63,109,129],"present":[64],"a":[65,83,132,189],"novel":[66,133],"method":[67,168],"to":[68,98,122],"estimate":[69],"dimensional":[70,175],"emotion":[71],"states,":[72],"where":[73],"color,":[74,151],"depth,":[75,152],"and":[76,113,153,186,188],"thermal":[77,114,154],"are":[80],"used":[81],"as":[82,116,141],"input.":[85],"Our":[86],"networks,":[87],"called":[88],"recurrent":[90],"attention":[91,96],"(MRAN),":[93],"learn":[94],"spatiotemporal":[95],"volumes":[97],"robustly":[99],"recognize":[100],"attention-boosted":[106],"feature":[107],"volumes.":[108],"leverage":[110],"depth":[112],"sequences":[115],"guidance":[117],"priors":[118],"sequence":[121],"selectively":[123],"focus":[124],"emotional":[126],"discriminative":[127],"regions.":[128],"also":[130],"introduce":[131],"benchmark":[134],"recognition,":[139],"termed":[140],"arousal-valence":[143,160],"(MAVFER),":[147],"which":[148],"consists":[149],"of":[150],"corresponding":[158],"continuous":[159],"scores.":[161],"The":[162],"experimental":[163],"results":[164,173],"show":[165],"that":[166],"our":[167],"achieve":[170],"in":[174],"datasets":[182],"including":[183,193],"RECOLA,":[184],"SEWA":[185],"AFEW,":[187],"dataset":[192],"MAVFER.":[194]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":23},{"year":2023,"cited_by_count":20},{"year":2022,"cited_by_count":18},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}