{"id":"https://openalex.org/W4399768306","doi":"https://doi.org/10.1109/taffc.2024.3415625","title":"Multimodal Sentimental Privileged Information Embedding for Improving Facial Expression Recognition","display_name":"Multimodal Sentimental Privileged Information Embedding for Improving Facial Expression Recognition","publication_year":2024,"publication_date":"2024-06-18","ids":{"openalex":"https://openalex.org/W4399768306","doi":"https://doi.org/10.1109/taffc.2024.3415625"},"language":"en","primary_location":{"id":"doi:10.1109/taffc.2024.3415625","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taffc.2024.3415625","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041345011","display_name":"Ning Sun","orcid":"https://orcid.org/0000-0002-6907-3756"},"institutions":[{"id":"https://openalex.org/I41198531","display_name":"Nanjing University of Posts and Telecommunications","ror":"https://ror.org/043bpky34","country_code":"CN","type":"education","lineage":["https://openalex.org/I41198531"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Ning Sun","raw_affiliation_strings":["College of Communication and Information Engineering, Nanjing University of Posts and Telecommunications, Nanjing, China","Engineering Research Center of Wideband Wireless Communication Technology Ministry of Education, Nanjing University of Posts and Telecommunications, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"College of Communication and Information Engineering, Nanjing University of Posts and Telecommunications, Nanjing, China","institution_ids":["https://openalex.org/I41198531"]},{"raw_affiliation_string":"Engineering Research Center of Wideband Wireless Communication Technology Ministry of Education, Nanjing University of Posts and Telecommunications, Nanjing, China","institution_ids":["https://openalex.org/I41198531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100596609","display_name":"Changwei You","orcid":null},"institutions":[{"id":"https://openalex.org/I41198531","display_name":"Nanjing University of Posts and Telecommunications","ror":"https://ror.org/043bpky34","country_code":"CN","type":"education","lineage":["https://openalex.org/I41198531"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Changwei You","raw_affiliation_strings":["College of Communication and Information Engineering, Nanjing University of Posts and Telecommunications, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"College of Communication and Information Engineering, Nanjing University of Posts and Telecommunications, Nanjing, China","institution_ids":["https://openalex.org/I41198531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100518719","display_name":"Wenming Zheng","orcid":null},"institutions":[{"id":"https://openalex.org/I76569877","display_name":"Southeast University","ror":"https://ror.org/04ct4d772","country_code":"CN","type":"education","lineage":["https://openalex.org/I76569877"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenming Zheng","raw_affiliation_strings":["Key Laboratory of Child Development and Learning Science, Ministry of Education, Research Center for Learning Science, Southeast University, Nanjing, Jiangsu, China","Key Laboratory of Child Development and Learning Science(Southeast University) Ministry of Education, Research Center for Learning Science, Southeast University, Nanjing, Jiangsu, China"],"affiliations":[{"raw_affiliation_string":"Key Laboratory of Child Development and Learning Science, Ministry of Education, Research Center for Learning Science, Southeast University, Nanjing, Jiangsu, China","institution_ids":["https://openalex.org/I76569877"]},{"raw_affiliation_string":"Key Laboratory of Child Development and Learning Science(Southeast University) Ministry of Education, Research Center for Learning Science, Southeast University, Nanjing, Jiangsu, China","institution_ids":["https://openalex.org/I76569877"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081669641","display_name":"Jixin Liu","orcid":"https://orcid.org/0000-0001-8414-4199"},"institutions":[{"id":"https://openalex.org/I41198531","display_name":"Nanjing University of Posts and Telecommunications","ror":"https://ror.org/043bpky34","country_code":"CN","type":"education","lineage":["https://openalex.org/I41198531"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jixin Liu","raw_affiliation_strings":["College of Communication and Information Engineering, Nanjing University of Posts and Telecommunications, Nanjing, China","Engineering Research Center of Wideband Wireless Communication Technology Ministry of Education, Nanjing University of Posts and Telecommunications, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"College of Communication and Information Engineering, Nanjing University of Posts and Telecommunications, Nanjing, China","institution_ids":["https://openalex.org/I41198531"]},{"raw_affiliation_string":"Engineering Research Center of Wideband Wireless Communication Technology Ministry of Education, Nanjing University of Posts and Telecommunications, Nanjing, China","institution_ids":["https://openalex.org/I41198531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024002089","display_name":"Lei Chai","orcid":"https://orcid.org/0000-0003-2110-838X"},"institutions":[{"id":"https://openalex.org/I41198531","display_name":"Nanjing University of Posts and Telecommunications","ror":"https://ror.org/043bpky34","country_code":"CN","type":"education","lineage":["https://openalex.org/I41198531"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei Chai","raw_affiliation_strings":["College of Communication and Information Engineering, Nanjing University of Posts and Telecommunications, Nanjing, China","Engineering Research Center of Wideband Wireless Communication Technology Ministry of Education, Nanjing University of Posts and Telecommunications, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"College of Communication and Information Engineering, Nanjing University of Posts and Telecommunications, Nanjing, China","institution_ids":["https://openalex.org/I41198531"]},{"raw_affiliation_string":"Engineering Research Center of Wideband Wireless Communication Technology Ministry of Education, Nanjing University of Posts and Telecommunications, Nanjing, China","institution_ids":["https://openalex.org/I41198531"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036049204","display_name":"Haian Sun","orcid":null},"institutions":[{"id":"https://openalex.org/I41198531","display_name":"Nanjing University of Posts and Telecommunications","ror":"https://ror.org/043bpky34","country_code":"CN","type":"education","lineage":["https://openalex.org/I41198531"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haian Sun","raw_affiliation_strings":["College of Communication and Information Engineering, Nanjing University of Posts and Telecommunications, Nanjing, China","Engineering Research Center of Wideband Wireless Communication Technology Ministry of Education, Nanjing University of Posts and Telecommunications, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"College of Communication and Information Engineering, Nanjing University of Posts and Telecommunications, Nanjing, China","institution_ids":["https://openalex.org/I41198531"]},{"raw_affiliation_string":"Engineering Research Center of Wideband Wireless Communication Technology Ministry of Education, Nanjing University of Posts and Telecommunications, Nanjing, China","institution_ids":["https://openalex.org/I41198531"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5041345011"],"corresponding_institution_ids":["https://openalex.org/I41198531"],"apc_list":null,"apc_paid":null,"fwci":3.1873,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.91711577,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"16","issue":"1","first_page":"133","last_page":"144"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.8303999900817871,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.8303999900817871,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.7494000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/facial-expression","display_name":"Facial expression","score":0.7019585371017456},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.6718430519104004},{"id":"https://openalex.org/keywords/expression","display_name":"Expression (computer science)","score":0.580167829990387},{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"Emotion recognition","score":0.5101190209388733},{"id":"https://openalex.org/keywords/facial-expression-recognition","display_name":"Facial expression recognition","score":0.4722541868686676},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.47053584456443787},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.46349382400512695},{"id":"https://openalex.org/keywords/affective-computing","display_name":"Affective computing","score":0.4101235270500183},{"id":"https://openalex.org/keywords/facial-recognition-system","display_name":"Facial recognition system","score":0.3932327926158905},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.3834390640258789},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.3637757897377014},{"id":"https://openalex.org/keywords/cognitive-science","display_name":"Cognitive science","score":0.3460635542869568},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.33771124482154846},{"id":"https://openalex.org/keywords/aesthetics","display_name":"Aesthetics","score":0.33555346727371216},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.33264100551605225},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.32116633653640747},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3202133774757385},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.14065110683441162}],"concepts":[{"id":"https://openalex.org/C195704467","wikidata":"https://www.wikidata.org/wiki/Q327968","display_name":"Facial expression","level":2,"score":0.7019585371017456},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.6718430519104004},{"id":"https://openalex.org/C90559484","wikidata":"https://www.wikidata.org/wiki/Q778379","display_name":"Expression (computer science)","level":2,"score":0.580167829990387},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.5101190209388733},{"id":"https://openalex.org/C2987714656","wikidata":"https://www.wikidata.org/wiki/Q1185804","display_name":"Facial expression recognition","level":4,"score":0.4722541868686676},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47053584456443787},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.46349382400512695},{"id":"https://openalex.org/C6438553","wikidata":"https://www.wikidata.org/wiki/Q1185804","display_name":"Affective computing","level":2,"score":0.4101235270500183},{"id":"https://openalex.org/C31510193","wikidata":"https://www.wikidata.org/wiki/Q1192553","display_name":"Facial recognition system","level":3,"score":0.3932327926158905},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3834390640258789},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3637757897377014},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.3460635542869568},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.33771124482154846},{"id":"https://openalex.org/C107038049","wikidata":"https://www.wikidata.org/wiki/Q35986","display_name":"Aesthetics","level":1,"score":0.33555346727371216},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.33264100551605225},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.32116633653640747},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3202133774757385},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.14065110683441162},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/taffc.2024.3415625","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taffc.2024.3415625","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W2067562626","https://openalex.org/W2145310492","https://openalex.org/W2146334809","https://openalex.org/W2765280073","https://openalex.org/W2799041689","https://openalex.org/W2883409523","https://openalex.org/W2904483377","https://openalex.org/W2964276171","https://openalex.org/W2982070372","https://openalex.org/W3034849760","https://openalex.org/W3091862369","https://openalex.org/W3171536535","https://openalex.org/W3179103990","https://openalex.org/W3194672963","https://openalex.org/W3197634753","https://openalex.org/W3203381531","https://openalex.org/W4226344365","https://openalex.org/W4245343722","https://openalex.org/W4294068743","https://openalex.org/W4312622804","https://openalex.org/W4321151193","https://openalex.org/W4321439011","https://openalex.org/W6631190155","https://openalex.org/W6719667659","https://openalex.org/W6784333009","https://openalex.org/W6799510764"],"related_works":["https://openalex.org/W4205986151","https://openalex.org/W3080495370","https://openalex.org/W2355913164","https://openalex.org/W1153638794","https://openalex.org/W2168968280","https://openalex.org/W2116055069","https://openalex.org/W2584926856","https://openalex.org/W2162992774","https://openalex.org/W2075935902","https://openalex.org/W4298012442"],"abstract_inverted_index":{"Facial":[0],"expression":[1,80,183],"recognition":[2,81],"(FER)":[3],"has":[4],"always":[5],"been":[6],"one":[7],"of":[8,25,99,144,157,168,187,202,212,244,262],"the":[9,16,23,45,90,96,113,122,127,170,196,215,232,242,252,255],"key":[10],"task":[11,129,234],"in":[12,86,214,251],"affective":[13],"computing.":[14],"Over":[15],"years,":[17],"researchers":[18],"have":[19],"worked":[20],"to":[21,50,125,194],"improve":[22,51,241],"performance":[24,53],"FER":[26,52,103,128,213,233],"by":[27,54,104,161,247,270],"designing":[28],"models":[29],"with":[30,147],"more":[31],"powerful":[32],"feature":[33],"extraction,":[34],"embedding":[35,98,224],"attention":[36],"mechanism,":[37],"and":[38,62,207,209,218],"reconstructing":[39],"missing":[40],"information,":[41],"etc.":[42],"Different":[43],"from":[44],"paradigms":[46],"above,":[47],"we":[48,151],"attempt":[49],"using":[55,249,271],"multimodal":[56,75,100,109,171,203,225,263,272],"sentiment":[57,110,172,204,226,264],"data,":[58],"such":[59],"as":[60,64,228],"audio":[61],"text,":[63],"privileged":[65,76,229],"information":[66,77,230],"(PI)":[67],"for":[68,102,121,176],"facial":[69,79,182],"images.":[70,189],"To":[71,140],"this":[72,87,93,145],"end,":[73],"a":[74,134,153,166,181],"embedded":[78],"network":[82],"(MPI-FER)":[83],"is":[84,133,174],"proposed":[85,175,197,256],"paper.":[88],"During":[89,112],"training":[91,143,149,155],"phase,":[92,115,254],"model":[94,123,146],"achieves":[95],"PI":[97],"data":[101,227,274],"developing":[105],"cross-modality":[106],"translation":[107],"between":[108],"data.":[111],"test":[114,253],"input":[116],"images":[117,238],"alone":[118],"are":[119],"sufficient":[120],"inference":[124],"accomplish":[126],"input.":[130],"The":[131,220],"MPI-FER":[132],"large-scale,":[135],"heterogeneous":[136],"deep":[137],"neural":[138],"network.":[139],"achieve":[141,259],"effective":[142],"limited":[148],"samples,":[150],"design":[152],"multi-stage":[154],"strategy":[156,167],"module-wise":[158],"pre-training":[159],"followed":[160],"end-to-end":[162],"fine-tuning.":[163],"In":[164],"addition,":[165],"filling":[169],"quaternion":[173],"implementing":[177],"our":[178],"method":[179,198,257],"on":[180,199,236],"database":[184],"consisting":[185],"only":[186,248],"face":[188,237],"We":[190],"conducted":[191],"extensive":[192],"experiments":[193],"evaluate":[195],"two":[200,210],"databases":[201,211],"analysis":[205,265],"(CH-SIMS":[206],"CMU-MOSI)":[208],"wild":[216],"(RAF-DB":[217],"AffectNet).":[219],"results":[221,261],"show":[222],"that":[223],"into":[231],"based":[235],"can":[239,258],"significantly":[240],"accuracy":[243],"FER.":[245],"Furthermore,":[246],"image":[250],"better":[260],"than":[266],"those":[267],"methods":[268],"achieved":[269],"sentimental":[273],"fusion.":[275]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":1}],"updated_date":"2025-12-26T23:08:49.675405","created_date":"2025-10-10T00:00:00"}