{"id":"https://openalex.org/W4405778930","doi":"https://doi.org/10.1109/tmm.2024.3521788","title":"FER-Former: Multimodal Transformer for Facial Expression Recognition","display_name":"FER-Former: Multimodal Transformer for Facial Expression Recognition","publication_year":2024,"publication_date":"2024-12-25","ids":{"openalex":"https://openalex.org/W4405778930","doi":"https://doi.org/10.1109/tmm.2024.3521788"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2024.3521788","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2024.3521788","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040412882","display_name":"Yande Li","orcid":"https://orcid.org/0000-0002-9972-8020"},"institutions":[{"id":"https://openalex.org/I76214153","display_name":"Lanzhou University","ror":"https://ror.org/01mkqqe32","country_code":"CN","type":"education","lineage":["https://openalex.org/I76214153"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yande Li","raw_affiliation_strings":["School of Information Science and Engineering, Lanzhou University, Lanzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-9972-8020","affiliations":[{"raw_affiliation_string":"School of Information Science and Engineering, Lanzhou University, Lanzhou, China","institution_ids":["https://openalex.org/I76214153"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100634350","display_name":"Mingjie Wang","orcid":"https://orcid.org/0000-0002-7346-1110"},"institutions":[{"id":"https://openalex.org/I1328775524","display_name":"Zhejiang Sci-Tech University","ror":"https://ror.org/03893we55","country_code":"CN","type":"education","lineage":["https://openalex.org/I1328775524"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingjie Wang","raw_affiliation_strings":["School of Science, Zhejiang Sci-Tech University, Hangzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Science, Zhejiang Sci-Tech University, Hangzhou, China","institution_ids":["https://openalex.org/I1328775524"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109011312","display_name":"Minglun Gong","orcid":"https://orcid.org/0000-0001-5820-5381"},"institutions":[{"id":"https://openalex.org/I79817857","display_name":"University of Guelph","ror":"https://ror.org/01r7awg59","country_code":"CA","type":"education","lineage":["https://openalex.org/I79817857"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Minglun Gong","raw_affiliation_strings":["School of Computer Science, University of Guelph, Guelph, ON, Canada","School of Computer Science, University of Guelph, Guelph, Canada"],"raw_orcid":"https://orcid.org/0000-0001-5820-5381","affiliations":[{"raw_affiliation_string":"School of Computer Science, University of Guelph, Guelph, ON, Canada","institution_ids":["https://openalex.org/I79817857"]},{"raw_affiliation_string":"School of Computer Science, University of Guelph, Guelph, Canada","institution_ids":["https://openalex.org/I79817857"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082133960","display_name":"Yonggang Lu","orcid":"https://orcid.org/0000-0001-8926-2039"},"institutions":[{"id":"https://openalex.org/I76214153","display_name":"Lanzhou University","ror":"https://ror.org/01mkqqe32","country_code":"CN","type":"education","lineage":["https://openalex.org/I76214153"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yonggang Lu","raw_affiliation_strings":["School of Information Science and Engineering, Lanzhou University, Lanzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-8926-2039","affiliations":[{"raw_affiliation_string":"School of Information Science and Engineering, Lanzhou University, Lanzhou, China","institution_ids":["https://openalex.org/I76214153"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100418822","display_name":"Li Liu","orcid":"https://orcid.org/0000-0002-4776-5292"},"institutions":[{"id":"https://openalex.org/I158842170","display_name":"Chongqing University","ror":"https://ror.org/023rhb549","country_code":"CN","type":"education","lineage":["https://openalex.org/I158842170"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li Liu","raw_affiliation_strings":["School of Big Data and Software Engineering, Chongqing University, Chongqing, China"],"raw_orcid":"https://orcid.org/0000-0002-4776-5292","affiliations":[{"raw_affiliation_string":"School of Big Data and Software Engineering, Chongqing University, Chongqing, China","institution_ids":["https://openalex.org/I158842170"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2495,"currency":"USD","value_usd":2495},"apc_paid":null,"fwci":10.1871,"has_fulltext":false,"cited_by_count":30,"citation_normalized_percentile":{"value":0.98734938,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"27","issue":null,"first_page":"2412","last_page":"2422"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.7710999846458435,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.7710999846458435,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11398","display_name":"Hand Gesture Recognition Systems","score":0.7559000253677368,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10057","display_name":"Face and Expression Recognition","score":0.6707000136375427,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7149795293807983},{"id":"https://openalex.org/keywords/facial-expression-recognition","display_name":"Facial expression recognition","score":0.563390851020813},{"id":"https://openalex.org/keywords/facial-expression","display_name":"Facial expression","score":0.4790971577167511},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.4660566747188568},{"id":"https://openalex.org/keywords/facial-recognition-system","display_name":"Facial recognition system","score":0.43082353472709656},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.41764742136001587},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3989807665348053},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.3799576759338379},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3644375503063202},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.1316605508327484},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.10346299409866333},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10253649950027466}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7149795293807983},{"id":"https://openalex.org/C2987714656","wikidata":"https://www.wikidata.org/wiki/Q1185804","display_name":"Facial expression recognition","level":4,"score":0.563390851020813},{"id":"https://openalex.org/C195704467","wikidata":"https://www.wikidata.org/wiki/Q327968","display_name":"Facial expression","level":2,"score":0.4790971577167511},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4660566747188568},{"id":"https://openalex.org/C31510193","wikidata":"https://www.wikidata.org/wiki/Q1192553","display_name":"Facial recognition system","level":3,"score":0.43082353472709656},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.41764742136001587},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3989807665348053},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.3799576759338379},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3644375503063202},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.1316605508327484},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.10346299409866333},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10253649950027466}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2024.3521788","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2024.3521788","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6136472032","display_name":null,"funder_award_id":"202006180033","funder_id":"https://openalex.org/F4320322725","funder_display_name":"China Scholarship Council"}],"funders":[{"id":"https://openalex.org/F4320322725","display_name":"China Scholarship Council","ror":"https://ror.org/04atp4p48"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":67,"referenced_works":["https://openalex.org/W2035372623","https://openalex.org/W2041616772","https://openalex.org/W2081835714","https://openalex.org/W2083021723","https://openalex.org/W2103943262","https://openalex.org/W2341528187","https://openalex.org/W2481681431","https://openalex.org/W2515770085","https://openalex.org/W2889978276","https://openalex.org/W2896277673","https://openalex.org/W2904483377","https://openalex.org/W2905312821","https://openalex.org/W2912538417","https://openalex.org/W2963363102","https://openalex.org/W2963712289","https://openalex.org/W2969985801","https://openalex.org/W2982070372","https://openalex.org/W3003720578","https://openalex.org/W3006012949","https://openalex.org/W3034504038","https://openalex.org/W3035336958","https://openalex.org/W3039257983","https://openalex.org/W3092920534","https://openalex.org/W3118405827","https://openalex.org/W3127230150","https://openalex.org/W3136416617","https://openalex.org/W3143320354","https://openalex.org/W3154541964","https://openalex.org/W3171162369","https://openalex.org/W3172509117","https://openalex.org/W3173220247","https://openalex.org/W3173787706","https://openalex.org/W3175546442","https://openalex.org/W3178221902","https://openalex.org/W3179103990","https://openalex.org/W3180353325","https://openalex.org/W3185372235","https://openalex.org/W3195286673","https://openalex.org/W3201484072","https://openalex.org/W3202521030","https://openalex.org/W3211240005","https://openalex.org/W4210794570","https://openalex.org/W4214493665","https://openalex.org/W4214588794","https://openalex.org/W4283575658","https://openalex.org/W4283787135","https://openalex.org/W4298348845","https://openalex.org/W4304086144","https://openalex.org/W4304091652","https://openalex.org/W4312678820","https://openalex.org/W4312702383","https://openalex.org/W4312726009","https://openalex.org/W4312910992","https://openalex.org/W4312950730","https://openalex.org/W4312971576","https://openalex.org/W4313128368","https://openalex.org/W4316660726","https://openalex.org/W4385245566","https://openalex.org/W6632670727","https://openalex.org/W6755207826","https://openalex.org/W6784333009","https://openalex.org/W6789753369","https://openalex.org/W6790019176","https://openalex.org/W6791353385","https://openalex.org/W6798805250","https://openalex.org/W6798913250","https://openalex.org/W6800139874"],"related_works":["https://openalex.org/W4205986151","https://openalex.org/W2355913164","https://openalex.org/W2162992774","https://openalex.org/W2985118265","https://openalex.org/W1153638794","https://openalex.org/W2168968280","https://openalex.org/W2901126000","https://openalex.org/W4323520705","https://openalex.org/W2116055069","https://openalex.org/W2356663679"],"abstract_inverted_index":{"The":[0],"ever-increasing":[1],"demands":[2],"for":[3,51,137],"intuitive":[4],"interactions":[5],"in":[6,14,26,43,55,135],"virtual":[7],"reality":[8],"have":[9],"led":[10],"to":[11,59,63,101,106,126],"surging":[12],"interests":[13],"facial":[15,52],"expression":[16,53],"recognition":[17,54],"(FER).":[18],"There":[19],"are":[20,155],"however":[21],"several":[22],"issues":[23],"commonly":[24],"seen":[25],"existing":[27,175],"methods,":[28],"including":[29],"narrow":[30,68],"receptive":[31,69,149],"fields":[32,150],"and":[33,83,116,131],"homogenous":[34,91],"supervisory":[35,92],"signals.":[36],"To":[37,85],"address":[38,64],"these":[39],"issues,":[40],"we":[41],"propose":[42],"this":[44],"paper":[45],"a":[46,71,94,120],"novel":[47],"multimodal":[48,152],"supervision-steering":[49],"transformer":[50,122],"the":[56,65,88,112,142,167,170,174],"wild,":[57],"referred":[58],"as":[60],"FER-former.":[61],"Specifically,":[62],"limitation":[66],"of":[67,90,144,169],"fields,":[70],"hybrid":[72],"feature":[73],"extraction":[74],"pipeline":[75],"is":[76,99,124],"designed":[77],"by":[78],"cascading":[79],"both":[80],"prevailing":[81],"CNNs":[82],"transformers.":[84],"deal":[86],"with":[87,151],"issue":[89],"signals,":[93],"heterogeneous":[95],"domain-steering":[96],"supervision":[97],"module":[98],"proposed":[100,171],"incorporate":[102],"text-space":[103],"semantic":[104,153],"correlations":[105],"enhance":[107],"image":[108,115],"features,":[109],"based":[110],"on":[111,141,163],"similarity":[113],"between":[114],"text":[117],"features.":[118],"Additionally,":[119],"FER-specific":[121],"encoder":[123],"introduced":[125],"characterize":[127],"conventional":[128],"one-hot":[129],"label-focusing":[130],"CLIP-based":[132],"text-oriented":[133],"tokens":[134],"parallel":[136],"final":[138],"classification.":[139],"Based":[140],"collaboration":[143],"multifarious":[145],"token":[146],"heads,":[147],"global":[148],"cues":[154],"captured,":[156],"delivering":[157],"superb":[158],"learning":[159],"capability.":[160],"Extensive":[161],"experiments":[162],"popular":[164],"benchmarks":[165],"demonstrate":[166],"superiority":[168],"FER-former":[172],"over":[173],"state-of-the-art":[176],"methods.":[177]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":23}],"updated_date":"2026-08-29T07:29:34.045763","created_date":"2025-10-10T00:00:00"}