{"id":"https://openalex.org/W4400527348","doi":"https://doi.org/10.1109/fg59268.2024.10581957","title":"EAT-Face: Emotion-Controllable Audio-Driven Talking Face Generation via Diffusion Model","display_name":"EAT-Face: Emotion-Controllable Audio-Driven Talking Face Generation via Diffusion Model","publication_year":2024,"publication_date":"2024-05-27","ids":{"openalex":"https://openalex.org/W4400527348","doi":"https://doi.org/10.1109/fg59268.2024.10581957"},"language":"en","primary_location":{"id":"doi:10.1109/fg59268.2024.10581957","is_oa":false,"landing_page_url":"https://doi.org/10.1109/fg59268.2024.10581957","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE 18th International Conference on Automatic Face and Gesture Recognition (FG)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034690828","display_name":"Haodi Wang","orcid":"https://orcid.org/0000-0002-6970-9291"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haodi Wang","raw_affiliation_strings":["Sun Yat-sen University,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sun Yat-sen University,China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084784341","display_name":"Xiaojun Jia","orcid":"https://orcid.org/0000-0002-2018-9344"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Xiaojun Jia","raw_affiliation_strings":["Nanyang Technological University,Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanyang Technological University,Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068837264","display_name":"Xiaochun Cao","orcid":"https://orcid.org/0000-0001-7141-708X"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaochun Cao","raw_affiliation_strings":["Sun Yat-sen University,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sun Yat-sen University,China","institution_ids":["https://openalex.org/I157773358"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.0517,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.81463871,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9742000102996826,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9233999848365784,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/face","display_name":"Face (sociological concept)","score":0.7225189805030823},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6123501658439636},{"id":"https://openalex.org/keywords/diffusion","display_name":"Diffusion","score":0.5986213684082031},{"id":"https://openalex.org/keywords/face-to-face","display_name":"Face-to-face","score":0.46174508333206177},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.39524954557418823},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.3292929530143738},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3214704096317291},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.06830301880836487}],"concepts":[{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.7225189805030823},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6123501658439636},{"id":"https://openalex.org/C69357855","wikidata":"https://www.wikidata.org/wiki/Q163214","display_name":"Diffusion","level":2,"score":0.5986213684082031},{"id":"https://openalex.org/C2777359053","wikidata":"https://www.wikidata.org/wiki/Q2463823","display_name":"Face-to-face","level":2,"score":0.46174508333206177},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.39524954557418823},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.3292929530143738},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3214704096317291},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.06830301880836487},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/fg59268.2024.10581957","is_oa":false,"landing_page_url":"https://doi.org/10.1109/fg59268.2024.10581957","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE 18th International Conference on Automatic Face and Gesture Recognition (FG)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G8726916336","display_name":null,"funder_award_id":"62025604,U2336208","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W304855073","https://openalex.org/W2386796262","https://openalex.org/W4213235153","https://openalex.org/W2070454487","https://openalex.org/W4388505810","https://openalex.org/W411361004","https://openalex.org/W2560105438","https://openalex.org/W2612607254","https://openalex.org/W650744398","https://openalex.org/W2248050490"],"abstract_inverted_index":{"Audio-driven":[0,48],"talking":[1,202],"face":[2,98,117,203],"generation":[3,51],"is":[4,175],"a":[5,9,70,76,140,158],"promising":[6],"task":[7],"with":[8,101],"lot":[10],"of":[11,33,62,69,111,194],"attention.":[12],"Despite":[13],"abundant":[14],"efforts":[15],"are":[16],"devoted":[17],"to":[18,58,165,177],"video":[19],"quality":[20],"and":[21,75,90,116,148,188,200],"lip":[22],"synchronization,":[23],"most":[24],"existing":[25],"works":[26],"do":[27],"not":[28],"take":[29],"the":[30,65,125,132,146,167,172,192],"unignorable":[31],"aspect":[32],"facial":[34,126],"emotional":[35,201],"expression":[36],"into":[37],"account":[38],"during":[39],"generation.":[40],"In":[41,169],"this":[42,153,170],"paper,":[43],"we":[44,155],"propose":[45,157],"an":[46],"Emotion-controllable":[47],"Talking":[49,71],"Face":[50,72],"framework":[52,196],"called":[53,160],"EAT-Face":[54,174],"that":[55],"enables":[56],"us":[57],"control":[59,178],"multiple":[60],"types":[61],"emotions.":[63],"Specifically,":[64],"proposed":[66,173],"method":[67],"consists":[68],"Reconstructor":[73],"(TFR)":[74],"Facial":[77],"Emotion":[78],"Controller":[79],"(FEC),":[80],"utilizing":[81],"fused":[82],"multimodal":[83],"information":[84],"including":[85],"audio":[86,112],"signals,":[87],"visual":[88],"images,":[89],"textual":[91],"emotions":[92,127],"for":[93],"synthesis.":[94],"Firstly,":[95],"TFR":[96],"predicts":[97],"images":[99],"synchronized":[100],"given":[102],"audios":[103],"from":[104,136],"random":[105],"noises,":[106],"leveraging":[107,131],"external":[108],"guidances":[109],"comprised":[110],"features,":[113],"character":[114,149],"references,":[115],"masks":[118],"as":[119],"conditions.":[120],"Then,":[121],"FEC":[122],"further":[123],"manipulates":[124],"based":[128],"on":[129],"TFR,":[130],"emotion":[133,137,179],"embeddings":[134],"extracted":[135],"texts.":[138],"However,":[139],"semantic":[141],"misalignment":[142],"problem":[143],"lies":[144],"in":[145,197],"emotion-texts":[147],"images.":[150],"To":[151],"tackle":[152],"issue,":[154],"additionally":[156],"strategy":[159],"joint":[161],"Emotion-Visual":[162],"Embedding":[163],"(EVE)":[164],"mitigate":[166],"misalignment.":[168],"way,":[171],"captive":[176],"more":[180],"precisely.":[181],"Extensive":[182],"experiments":[183],"involving":[184],"both":[185],"objective":[186],"evaluations":[187],"subjective":[189],"investigations":[190],"demonstrate":[191],"effectiveness":[193],"our":[195],"synthesizing":[198],"high-fidelity":[199],"videos.":[204]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}