{"id":"https://openalex.org/W4405811810","doi":"https://doi.org/10.1109/tcsvt.2024.3523359","title":"Multimodal Emotional Talking Face Generation Based on Action Units","display_name":"Multimodal Emotional Talking Face Generation Based on Action Units","publication_year":2024,"publication_date":"2024-12-26","ids":{"openalex":"https://openalex.org/W4405811810","doi":"https://doi.org/10.1109/tcsvt.2024.3523359"},"language":"en","primary_location":{"id":"doi:10.1109/tcsvt.2024.3523359","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcsvt.2024.3523359","pdf_url":null,"source":{"id":"https://openalex.org/S115173108","display_name":"IEEE Transactions on Circuits and Systems for Video Technology","issn_l":"1051-8215","issn":["1051-8215","1558-2205"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems for Video Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103214571","display_name":"Jiayi Lyu","orcid":"https://orcid.org/0009-0007-9202-0947"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiayi Lyu","raw_affiliation_strings":["School of Engineering Science, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0007-9202-0947","affiliations":[{"raw_affiliation_string":"School of Engineering Science, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056446118","display_name":"Xing Lan","orcid":"https://orcid.org/0000-0003-3624-5926"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xing Lan","raw_affiliation_strings":["School of Engineering Science, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3624-5926","affiliations":[{"raw_affiliation_string":"School of Engineering Science, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017022799","display_name":"Guohong Hu","orcid":"https://orcid.org/0000-0002-2980-5166"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guohong Hu","raw_affiliation_strings":["School of Engineering Science, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Engineering Science, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005793252","display_name":"Hanyu Jiang","orcid":"https://orcid.org/0009-0003-9258-521X"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hanyu Jiang","raw_affiliation_strings":["School of Engineering Science, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0003-9258-521X","affiliations":[{"raw_affiliation_string":"School of Engineering Science, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102495226","display_name":"Wei Gan","orcid":null},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Gan","raw_affiliation_strings":["School of Engineering Science, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Engineering Science, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101698854","display_name":"Jinbao Wang","orcid":"https://orcid.org/0000-0001-5916-8965"},"institutions":[{"id":"https://openalex.org/I180726961","display_name":"Shenzhen University","ror":"https://ror.org/01vy4gh70","country_code":"CN","type":"education","lineage":["https://openalex.org/I180726961"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jinbao Wang","raw_affiliation_strings":["National Engineering Laboratory for Big Data System Computing, Shenzhen University, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0001-5916-8965","affiliations":[{"raw_affiliation_string":"National Engineering Laboratory for Big Data System Computing, Shenzhen University, Shenzhen, China","institution_ids":["https://openalex.org/I180726961"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100763213","display_name":"Jian Xue","orcid":"https://orcid.org/0000-0002-9460-802X"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jian Xue","raw_affiliation_strings":["School of Engineering Science, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9460-802X","affiliations":[{"raw_affiliation_string":"School of Engineering Science, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2645,"currency":"USD","value_usd":2645},"apc_paid":null,"fwci":0.3104,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.55648198,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":"35","issue":"5","first_page":"4026","last_page":"4038"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.8105000257492065,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.8105000257492065,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.05180000141263008,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.0421999990940094,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6073482036590576},{"id":"https://openalex.org/keywords/face","display_name":"Face (sociological concept)","score":0.5788999795913696},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.5352606773376465},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.438101202249527},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.39584076404571533},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.38851675391197205},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.3290629982948303},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3200008273124695},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.16121557354927063}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6073482036590576},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.5788999795913696},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.5352606773376465},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.438101202249527},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.39584076404571533},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.38851675391197205},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3290629982948303},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3200008273124695},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.16121557354927063},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tcsvt.2024.3523359","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcsvt.2024.3523359","pdf_url":null,"source":{"id":"https://openalex.org/S115173108","display_name":"IEEE Transactions on Circuits and Systems for Video Technology","issn_l":"1051-8215","issn":["1051-8215","1558-2205"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems for Video Technology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1651903107","display_name":null,"funder_award_id":"2023B1212060076","funder_id":"https://openalex.org/F4320314147","funder_display_name":"Guangdong Provincial Key Laboratory of Urology"},{"id":"https://openalex.org/G2973092727","display_name":null,"funder_award_id":"62032022","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7148651236","display_name":null,"funder_award_id":"62236006","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8673813548","display_name":null,"funder_award_id":"62320106007","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320314147","display_name":"Guangdong Provincial Key Laboratory of Urology","ror":null},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":64,"referenced_works":["https://openalex.org/W2015143272","https://openalex.org/W2095499283","https://openalex.org/W2096342506","https://openalex.org/W2140351517","https://openalex.org/W2331128040","https://openalex.org/W2551572271","https://openalex.org/W2604379605","https://openalex.org/W2758030817","https://openalex.org/W2808631503","https://openalex.org/W2883861033","https://openalex.org/W2944294033","https://openalex.org/W2962714886","https://openalex.org/W2963466847","https://openalex.org/W2963847434","https://openalex.org/W2968614444","https://openalex.org/W3018564516","https://openalex.org/W3019952993","https://openalex.org/W3081492798","https://openalex.org/W3095664976","https://openalex.org/W3099284785","https://openalex.org/W3154807520","https://openalex.org/W3174724419","https://openalex.org/W3174763799","https://openalex.org/W3180770160","https://openalex.org/W3186090335","https://openalex.org/W3197199219","https://openalex.org/W3202128205","https://openalex.org/W3204680331","https://openalex.org/W3211147706","https://openalex.org/W4200295417","https://openalex.org/W4200631136","https://openalex.org/W4214626920","https://openalex.org/W4281730245","https://openalex.org/W4283818626","https://openalex.org/W4285599866","https://openalex.org/W4285601347","https://openalex.org/W4304779562","https://openalex.org/W4311137635","https://openalex.org/W4377235424","https://openalex.org/W4381785813","https://openalex.org/W4382240211","https://openalex.org/W4385764582","https://openalex.org/W4386072006","https://openalex.org/W4386072021","https://openalex.org/W4386075823","https://openalex.org/W4386076090","https://openalex.org/W4386363058","https://openalex.org/W4390872334","https://openalex.org/W4390872769","https://openalex.org/W4392824768","https://openalex.org/W4400573497","https://openalex.org/W4401807241","https://openalex.org/W4402982696","https://openalex.org/W4404199654","https://openalex.org/W6687566353","https://openalex.org/W6765779288","https://openalex.org/W6846992717","https://openalex.org/W6847553177","https://openalex.org/W6849966858","https://openalex.org/W6859566922","https://openalex.org/W6864518445","https://openalex.org/W6869123908","https://openalex.org/W6869353929","https://openalex.org/W6872758961"],"related_works":["https://openalex.org/W2084487854","https://openalex.org/W2353878298","https://openalex.org/W2077682749","https://openalex.org/W2298450300","https://openalex.org/W2374729771","https://openalex.org/W3200334421","https://openalex.org/W3122382218","https://openalex.org/W2738836896","https://openalex.org/W4310765266","https://openalex.org/W2352896083"],"abstract_inverted_index":{"Talking":[0,87],"face":[1],"generation":[2,101],"focuses":[3],"on":[4,25,59,109,215,230],"creating":[5],"natural":[6],"facial":[7,26,95,110,122,143],"animations":[8],"that":[9,106],"align":[10],"with":[11,88,203],"the":[12,41,100,139,216,220,231],"provided":[13],"text":[14],"or":[15],"audio":[16],"input.":[17],"Current":[18],"methods":[19],"in":[20,39,64],"this":[21,57,78],"field":[22],"primarily":[23],"rely":[24,108],"landmarks":[27,61,118],"to":[28,51,119,197],"convey":[29],"emotional":[30,48,74,174],"changes.":[31],"However,":[32],"spatial":[33,54],"key-points":[34],"are":[35],"valuable,":[36],"yet":[37],"limited":[38],"capturing":[40,138],"intricate":[42],"dynamics":[43],"and":[44,67,117,132,159,162,173,187,207],"subtle":[45],"nuances":[46],"of":[47,135,219,228],"expressions":[49,123],"due":[50],"their":[52],"restricted":[53],"coverage.":[55],"Consequently,":[56],"reliance":[58],"sparse":[60],"can":[62],"result":[63],"decreased":[65],"accuracy":[66,218,227],"visual":[68,205],"quality,":[69],"especially":[70],"when":[71],"representing":[72],"complex":[73,140],"states.":[75],"To":[76],"address":[77],"issue,":[79],"we":[80],"propose":[81],"a":[82,130,149,212],"novel":[83],"method":[84,128],"called":[85],"Emotional":[86],"Action":[89,96,115],"Unit":[90],"(ETAU),":[91],"which":[92],"seamlessly":[93,153],"integrates":[94],"Units":[97,116],"(AUs)":[98],"into":[99],"process.":[102],"Unlike":[103],"previous":[104,191],"works":[105],"solely":[107],"landmarks,":[111],"ETAU":[112,147,189,210],"employs":[113],"both":[114],"comprehensively":[120],"represent":[121],"through":[124],"interpretable":[125],"representations.":[126],"Our":[127],"provides":[129],"detailed":[131],"dynamic":[133],"representation":[134],"emotions":[136],"by":[137,152,163],"interactions":[141],"among":[142],"muscle":[144],"movements.":[145],"Moreover,":[146,209],"adopts":[148],"multi-modal":[150],"strategy":[151],"integrating":[154],"emotion":[155,217],"prompts,":[156],"driving":[157],"videos,":[158],"target":[160],"images,":[161],"leveraging":[164],"various":[165],"input":[166],"data":[167],"effectively,":[168],"it":[169],"generates":[170],"highly":[171],"realistic":[172],"talking-face":[175],"videos.":[176],"Through":[177],"extensive":[178],"evaluations":[179],"across":[180],"multiple":[181],"datasets,":[182],"including":[183],"MEAD,":[184],"LRW,":[185],"GRID":[186],"HDTF,":[188],"outperforms":[190],"methods,":[192],"showcasing":[193],"its":[194],"superior":[195],"ability":[196],"generate":[198],"high-quality,":[199],"expressive":[200],"talking":[201],"faces":[202],"improved":[204],"fidelity":[206],"synchronization.":[208],"exhibits":[211],"significant":[213],"improvement":[214],"generated":[221],"results,":[222],"reaching":[223],"an":[224],"impressive":[225],"average":[226],"84%":[229],"MEAD":[232],"dataset.":[233]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-10-10T00:00:00"}