{"id":"https://openalex.org/W4294982505","doi":"https://doi.org/10.1109/tim.2022.3204940","title":"A Novel Emotion-Aware Method Based on the Fusion of Textual Description of Speech, Body Movements, and Facial Expressions","display_name":"A Novel Emotion-Aware Method Based on the Fusion of Textual Description of Speech, Body Movements, and Facial Expressions","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4294982505","doi":"https://doi.org/10.1109/tim.2022.3204940"},"language":"en","primary_location":{"id":"doi:10.1109/tim.2022.3204940","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tim.2022.3204940","pdf_url":null,"source":{"id":"https://openalex.org/S10892749","display_name":"IEEE Transactions on Instrumentation and Measurement","issn_l":"0018-9456","issn":["0018-9456","1557-9662"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Instrumentation and Measurement","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026223895","display_name":"Guanglong Du","orcid":"https://orcid.org/0000-0001-9425-843X"},"institutions":[{"id":"https://openalex.org/I90610280","display_name":"South China University of Technology","ror":"https://ror.org/0530pts50","country_code":"CN","type":"education","lineage":["https://openalex.org/I90610280"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Guanglong Du","raw_affiliation_strings":["School of Computer Science and Engineering, South China University of Technology, Guangzhou, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, South China University of Technology, Guangzhou, China","institution_ids":["https://openalex.org/I90610280"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035440634","display_name":"Yuwen Zeng","orcid":"https://orcid.org/0000-0003-2895-0377"},"institutions":[{"id":"https://openalex.org/I90610280","display_name":"South China University of Technology","ror":"https://ror.org/0530pts50","country_code":"CN","type":"education","lineage":["https://openalex.org/I90610280"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuwen Zeng","raw_affiliation_strings":["School of Computer Science and Engineering, South China University of Technology, Guangzhou, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, South China University of Technology, Guangzhou, China","institution_ids":["https://openalex.org/I90610280"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055615305","display_name":"Kang Su","orcid":"https://orcid.org/0000-0002-3358-161X"},"institutions":[{"id":"https://openalex.org/I90610280","display_name":"South China University of Technology","ror":"https://ror.org/0530pts50","country_code":"CN","type":"education","lineage":["https://openalex.org/I90610280"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kang Su","raw_affiliation_strings":["School of Computer Science and Engineering, South China University of Technology, Guangzhou, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, South China University of Technology, Guangzhou, China","institution_ids":["https://openalex.org/I90610280"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100665304","display_name":"Chunquan Li","orcid":"https://orcid.org/0000-0002-5493-6379"},"institutions":[{"id":"https://openalex.org/I141649914","display_name":"Nanchang University","ror":"https://ror.org/042v6xz23","country_code":"CN","type":"education","lineage":["https://openalex.org/I141649914"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chunquan Li","raw_affiliation_strings":["School of Information Engineering, Nanchang University, Nanchang, China"],"affiliations":[{"raw_affiliation_string":"School of Information Engineering, Nanchang University, Nanchang, China","institution_ids":["https://openalex.org/I141649914"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100737125","display_name":"Xueqian Wang","orcid":"https://orcid.org/0000-0003-3542-0593"},"institutions":[{"id":"https://openalex.org/I3131625388","display_name":"University Town of Shenzhen","ror":"https://ror.org/05f5j6225","country_code":"CN","type":"education","lineage":["https://openalex.org/I3131625388"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xueqian Wang","raw_affiliation_strings":["Tsinghua Shenzhen International Graduate School, University Town of Shenzhen, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School, University Town of Shenzhen, Shenzhen, China","institution_ids":["https://openalex.org/I3131625388"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003969171","display_name":"Shaohua Teng","orcid":"https://orcid.org/0000-0002-7204-1288"},"institutions":[{"id":"https://openalex.org/I139024713","display_name":"Guangdong University of Technology","ror":"https://ror.org/04azbjn80","country_code":"CN","type":"education","lineage":["https://openalex.org/I139024713"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shaohua Teng","raw_affiliation_strings":["Department of Artificial Intelligent and Information Engineering, Guangdong University of Technology, Guangzhou, China"],"affiliations":[{"raw_affiliation_string":"Department of Artificial Intelligent and Information Engineering, Guangdong University of Technology, Guangzhou, China","institution_ids":["https://openalex.org/I139024713"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100446347","display_name":"Di Li","orcid":"https://orcid.org/0000-0001-5065-9565"},"institutions":[{"id":"https://openalex.org/I90610280","display_name":"South China University of Technology","ror":"https://ror.org/0530pts50","country_code":"CN","type":"education","lineage":["https://openalex.org/I90610280"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Di Li","raw_affiliation_strings":["School of Mechanical and Automotive Engineering, South China University of Technology, Guangzhou, China"],"affiliations":[{"raw_affiliation_string":"School of Mechanical and Automotive Engineering, South China University of Technology, Guangzhou, China","institution_ids":["https://openalex.org/I90610280"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100659849","display_name":"Peter Liu","orcid":"https://orcid.org/0000-0002-8703-6967"},"institutions":[{"id":"https://openalex.org/I67031392","display_name":"Carleton University","ror":"https://ror.org/02qtvee93","country_code":"CA","type":"education","lineage":["https://openalex.org/I67031392"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Peter Xiaoping Liu","raw_affiliation_strings":["Department of Systems and Computer Engineering, Carleton University, Ottawa, Canada"],"affiliations":[{"raw_affiliation_string":"Department of Systems and Computer Engineering, Carleton University, Ottawa, Canada","institution_ids":["https://openalex.org/I67031392"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5026223895"],"corresponding_institution_ids":["https://openalex.org/I90610280"],"apc_list":null,"apc_paid":null,"fwci":2.6744,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.8999219,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"71","issue":null,"first_page":"1","last_page":"16"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13731","display_name":"Advanced Computing and Algorithms","score":0.9394000172615051,"subfield":{"id":"https://openalex.org/subfields/3322","display_name":"Urban Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9388999938964844,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7599606513977051},{"id":"https://openalex.org/keywords/facial-expression","display_name":"Facial expression","score":0.6297434568405151},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5977014303207397},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.5342195630073547},{"id":"https://openalex.org/keywords/fuse","display_name":"Fuse (electrical)","score":0.5176966190338135},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5130094885826111},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.41409018635749817},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.40015727281570435},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.35301271080970764}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7599606513977051},{"id":"https://openalex.org/C195704467","wikidata":"https://www.wikidata.org/wiki/Q327968","display_name":"Facial expression","level":2,"score":0.6297434568405151},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5977014303207397},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5342195630073547},{"id":"https://openalex.org/C141353440","wikidata":"https://www.wikidata.org/wiki/Q182221","display_name":"Fuse (electrical)","level":2,"score":0.5176966190338135},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5130094885826111},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.41409018635749817},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.40015727281570435},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.35301271080970764},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tim.2022.3204940","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tim.2022.3204940","pdf_url":null,"source":{"id":"https://openalex.org/S10892749","display_name":"IEEE Transactions on Instrumentation and Measurement","issn_l":"0018-9456","issn":["0018-9456","1557-9662"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Instrumentation and Measurement","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6499999761581421,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G4192698891","display_name":null,"funder_award_id":"2022A1515010140","funder_id":"https://openalex.org/F4320337111","funder_display_name":"Basic and Applied Basic Research Foundation of Guangdong Province"},{"id":"https://openalex.org/G8063869721","display_name":null,"funder_award_id":"61973126","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320337111","display_name":"Basic and Applied Basic Research Foundation of Guangdong Province","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":73,"referenced_works":["https://openalex.org/W216108723","https://openalex.org/W579975518","https://openalex.org/W1129731247","https://openalex.org/W1527772862","https://openalex.org/W1564084013","https://openalex.org/W1606749791","https://openalex.org/W1972221662","https://openalex.org/W1997453653","https://openalex.org/W2012758188","https://openalex.org/W2016885041","https://openalex.org/W2023937851","https://openalex.org/W2030407775","https://openalex.org/W2034620660","https://openalex.org/W2039262954","https://openalex.org/W2095705004","https://openalex.org/W2129440600","https://openalex.org/W2131123711","https://openalex.org/W2144561273","https://openalex.org/W2158363765","https://openalex.org/W2168465881","https://openalex.org/W2206308392","https://openalex.org/W2321079037","https://openalex.org/W2342983306","https://openalex.org/W2525648609","https://openalex.org/W2588300744","https://openalex.org/W2605111162","https://openalex.org/W2765291577","https://openalex.org/W2765433083","https://openalex.org/W2783433009","https://openalex.org/W2790404832","https://openalex.org/W2800651388","https://openalex.org/W2807126412","https://openalex.org/W2889361304","https://openalex.org/W2889466822","https://openalex.org/W2901027086","https://openalex.org/W2910165986","https://openalex.org/W2915977493","https://openalex.org/W2941914178","https://openalex.org/W2963631961","https://openalex.org/W2964290400","https://openalex.org/W2979656641","https://openalex.org/W2982126608","https://openalex.org/W3005450949","https://openalex.org/W3014803340","https://openalex.org/W3017358282","https://openalex.org/W3035982241","https://openalex.org/W3039444588","https://openalex.org/W3047373554","https://openalex.org/W3047624172","https://openalex.org/W3093521632","https://openalex.org/W3094378471","https://openalex.org/W3118110335","https://openalex.org/W3119833370","https://openalex.org/W3121276084","https://openalex.org/W3126471623","https://openalex.org/W3126750668","https://openalex.org/W3127521573","https://openalex.org/W3128422390","https://openalex.org/W3129519514","https://openalex.org/W3203115521","https://openalex.org/W3209397829","https://openalex.org/W3215801214","https://openalex.org/W4213102761","https://openalex.org/W4230260685","https://openalex.org/W6627193292","https://openalex.org/W6631439307","https://openalex.org/W6636510571","https://openalex.org/W6674330103","https://openalex.org/W6682082992","https://openalex.org/W6769650007","https://openalex.org/W6770092901","https://openalex.org/W6793244244","https://openalex.org/W6794746887"],"related_works":["https://openalex.org/W3107474891","https://openalex.org/W2035372623","https://openalex.org/W2024390512","https://openalex.org/W3194078543","https://openalex.org/W2140205990","https://openalex.org/W2761544674","https://openalex.org/W2016461833","https://openalex.org/W1537282076","https://openalex.org/W4296713148","https://openalex.org/W2029005662"],"abstract_inverted_index":{"Emotion":[0],"computing":[1],"is":[2,36,251],"a":[3,14,47,70,105,118,171,175],"necessary":[4],"part":[5],"of":[6,13,59,77,80,91,102,145,158,165,194],"advanced":[7],"human\u2013computer":[8],"interaction.":[9],"An":[10],"appropriate":[11],"description":[12,79],"character\u2019s":[15,31],"facial":[16,85,96,146,204],"expressions,":[17,147],"body":[18,82,93,148],"languages,":[19],"and":[20,38,55,84,95,130,142,150,180,205,208,228,237,246],"speaking":[21],"styles":[22],"in":[23,225],"novels":[24],"always":[25],"enables":[26],"readers":[27],"to":[28,109,137,154,183,198,202,215,261],"infer":[29],"the":[30,41,57,61,75,89,139,162,166,217,222,229,257,265],"emotions.":[32,241],"Moreover,":[33],"multimodal":[34],"information":[35,42,103,201],"complementary":[37],"integrated.":[39],"Fusing":[40],"from":[43],"multiple":[44],"modes":[45],"into":[46,104],"textual":[48,78,156,163,212],"modal":[49],"can":[50,235,255],"get":[51],"better":[52],"fusion":[53,76],"results":[54,230],"overcome":[56],"bias":[58],"understanding":[60],"unimodal":[62,245,266],"information.":[63],"Inspired":[64],"by":[65,74,98,189,259],"these":[66],"facts,":[67],"we":[68,116],"develop":[69],"novel":[71],"emotion-aware":[72,223],"method":[73,196,250],"speech,":[81,92],"movements,":[83,94,149],"expression,":[86],"which":[87,152,254],"reduces":[88],"dimensionality":[90],"expressions":[97],"unifying":[99],"three":[100],"types":[101],"unified":[106],"component.":[107],"Specifically,":[108],"fuse":[110,216],"multimodel":[111],"features":[112,144,167],"for":[113],"emotion":[114],"recognition,":[115],"propose":[117],"two-stage":[119],"neural":[120,132,172],"network.":[121],"First,":[122],"bidirectional":[123],"long":[124],"short-term":[125],"memory-conditional":[126],"random":[127],"fields":[128],"(Bi-LSTM-CRF)":[129],"back-propagation":[131],"network":[133,173],"(BPNN)":[134],"are":[135,168,181,187,197],"used":[136,182],"analyze":[138],"extracted":[140],"vocal":[141],"visual":[143],"speeches,":[151],"aims":[153],"obtain":[155],"descriptions":[157,164],"different":[159],"features.":[160,218],"Second,":[161],"fused":[169],"through":[170],"with":[174,243,264],"self-organization":[176],"map":[177],"(SOM)":[178],"layer":[179],"compensate":[184],"layers":[185],"that":[186,232],"trained":[188],"web-based":[190],"corpus.":[191],"The":[192],"advantages":[193],"this":[195],"utilize":[199],"depth":[200],"track":[203],"bodily":[206],"movement,":[207],"employ":[209],"an":[210],"explainable":[211],"intermediate":[213],"representation":[214],"We":[219],"experimentally":[220],"tested":[221],"system":[224,234],"real-world":[226],"applications,":[227],"indicate":[231],"our":[233,249],"quickly":[236],"steadily":[238],"recognize":[239],"human":[240],"Compared":[242],"other":[244],"multimodal-fusion":[247],"algorithms,":[248],"more":[252],"precise,":[253],"improve":[256],"accuracy":[258],"up":[260],"30%":[262],"compared":[263],"method.":[267]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":3}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}