{"id":"https://openalex.org/W3006410788","doi":"https://doi.org/10.1109/tcsvt.2020.2973374","title":"Multimodal Inputs Driven Talking Face Generation With Spatial\u2013Temporal Dependency","display_name":"Multimodal Inputs Driven Talking Face Generation With Spatial\u2013Temporal Dependency","publication_year":2020,"publication_date":"2020-02-12","ids":{"openalex":"https://openalex.org/W3006410788","doi":"https://doi.org/10.1109/tcsvt.2020.2973374","mag":"3006410788"},"language":"en","primary_location":{"id":"doi:10.1109/tcsvt.2020.2973374","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcsvt.2020.2973374","pdf_url":null,"source":{"id":"https://openalex.org/S115173108","display_name":"IEEE Transactions on Circuits and Systems for Video Technology","issn_l":"1051-8215","issn":["1051-8215","1558-2205"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems for Video Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089799495","display_name":"Lingyun Yu","orcid":"https://orcid.org/0000-0001-6403-761X"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lingyun Yu","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0000-0001-6403-761X","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048818071","display_name":"Jun Yu","orcid":"https://orcid.org/0000-0002-3197-8103"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Yu","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0000-0002-3197-8103","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101933164","display_name":"Mengyan Li","orcid":"https://orcid.org/0000-0002-5475-1032"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mengyan Li","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007719477","display_name":"Qiang Ling","orcid":"https://orcid.org/0000-0001-5688-4130"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qiang Ling","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0000-0001-5688-4130","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I126520041"],"apc_list":null,"apc_paid":null,"fwci":4.7759,"has_fulltext":false,"cited_by_count":62,"citation_normalized_percentile":{"value":0.95770776,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"31","issue":"1","first_page":"203","last_page":"216"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.989799976348877,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7820354700088501},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6180307865142822},{"id":"https://openalex.org/keywords/dependency","display_name":"Dependency (UML)","score":0.5587575435638428},{"id":"https://openalex.org/keywords/face","display_name":"Face (sociological concept)","score":0.4661822021007538},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.43644633889198303},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4134889543056488},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3880782127380371}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7820354700088501},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6180307865142822},{"id":"https://openalex.org/C19768560","wikidata":"https://www.wikidata.org/wiki/Q320727","display_name":"Dependency (UML)","level":2,"score":0.5587575435638428},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.4661822021007538},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.43644633889198303},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4134889543056488},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3880782127380371},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tcsvt.2020.2973374","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcsvt.2020.2973374","pdf_url":null,"source":{"id":"https://openalex.org/S115173108","display_name":"IEEE Transactions on Circuits and Systems for Video Technology","issn_l":"1051-8215","issn":["1051-8215","1558-2205"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems for Video Technology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.44999998807907104,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G4938988039","display_name":null,"funder_award_id":"U1736123","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6039323323","display_name":"\u80fd\u8bf4\u4f1a\u5531\u7684\u4e09\u7ef4\u865a\u62df\u4eba\u5934\uff1a\u8bed\u97f3\u3001\u6b4c\u58f0\u548c\u60c5\u611f\u8bed\u4e49\u540c\u6b65\u7684\u4eba\u8138\u53ca\u53d1\u97f3\u5668\u5b98\u53ef\u89c6\u5316\u7814\u7a76","funder_award_id":"61572450","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":70,"referenced_works":["https://openalex.org/W34110307","https://openalex.org/W1499332833","https://openalex.org/W1524333225","https://openalex.org/W1569907127","https://openalex.org/W1686810756","https://openalex.org/W1836465849","https://openalex.org/W1916406603","https://openalex.org/W2049686551","https://openalex.org/W2064076387","https://openalex.org/W2093450784","https://openalex.org/W2099471712","https://openalex.org/W2115252128","https://openalex.org/W2132733106","https://openalex.org/W2145023731","https://openalex.org/W2301937176","https://openalex.org/W2331128040","https://openalex.org/W2486034530","https://openalex.org/W2560474170","https://openalex.org/W2570189907","https://openalex.org/W2593414223","https://openalex.org/W2598638573","https://openalex.org/W2613087992","https://openalex.org/W2619170298","https://openalex.org/W2737019897","https://openalex.org/W2738406145","https://openalex.org/W2780124704","https://openalex.org/W2782422271","https://openalex.org/W2794857359","https://openalex.org/W2804078698","https://openalex.org/W2804600264","https://openalex.org/W2806833697","https://openalex.org/W2899771611","https://openalex.org/W2904787397","https://openalex.org/W2944294033","https://openalex.org/W2949117887","https://openalex.org/W2962793481","https://openalex.org/W2962931900","https://openalex.org/W2963009026","https://openalex.org/W2963073614","https://openalex.org/W2963091558","https://openalex.org/W2963092440","https://openalex.org/W2963290645","https://openalex.org/W2963470893","https://openalex.org/W2963519453","https://openalex.org/W2963800363","https://openalex.org/W2963841322","https://openalex.org/W2964095416","https://openalex.org/W2964227577","https://openalex.org/W2964559396","https://openalex.org/W2965441391","https://openalex.org/W2969174745","https://openalex.org/W2996833883","https://openalex.org/W3102619627","https://openalex.org/W4285719527","https://openalex.org/W4320013936","https://openalex.org/W6601427873","https://openalex.org/W6631362777","https://openalex.org/W6637373629","https://openalex.org/W6638667902","https://openalex.org/W6677618333","https://openalex.org/W6729966448","https://openalex.org/W6741923574","https://openalex.org/W6747400857","https://openalex.org/W6748181857","https://openalex.org/W6749904982","https://openalex.org/W6751750676","https://openalex.org/W6752378368","https://openalex.org/W6753914649","https://openalex.org/W6756040250","https://openalex.org/W6770756366"],"related_works":["https://openalex.org/W2067317451","https://openalex.org/W4211085505","https://openalex.org/W2154771632","https://openalex.org/W2084758217","https://openalex.org/W3122478268","https://openalex.org/W408804804","https://openalex.org/W3086365953","https://openalex.org/W4226072953","https://openalex.org/W2392606101","https://openalex.org/W2134259672"],"abstract_inverted_index":{"Given":[0],"an":[1],"arbitrary":[2],"speech":[3],"clip":[4],"or":[5,39],"text":[6,40,130],"information":[7],"as":[8,41],"input,":[9,42],"the":[10,46,70,100,146,152,159,172,190,194],"proposed":[11,120,139],"work":[12],"aims":[13],"to":[14,121,140,157,170],"generate":[15,122,141,185],"a":[16,77,115,134,165],"talking":[17,101],"face":[18,65,91,102],"video":[19,112,142,187],"with":[20,36,76,126,189],"accurate":[21,123,197],"lip":[22,200],"synchronization.":[23],"Existing":[24],"works":[25],"mainly":[26],"have":[27],"three":[28],"limitations.":[29],"(1)":[30],"A":[31],"single-modal":[32],"learning":[33,117],"is":[34,53,67,119,138,155,168],"adopted":[35],"either":[37],"audio":[38],"hence":[43,56,81],"it":[44,57,82],"lacks":[45],"complementarity":[47],"ofmultimodal":[48],"inputs.":[49],"(2)":[50],"Each":[51,64],"frame":[52],"generated":[54,68],"independently,":[55],"ignores":[58],"thetemporal":[59],"dependencybetween":[60],"consecutive":[61],"frames.":[62],"(3)":[63],"image":[66,176],"by":[69],"traditional":[71],"convolution":[72],"neural":[73],"network":[74,135],"(CNN)":[75],"local":[78],"receptive":[79],"field,":[80],"cannot":[83],"effectively":[84],"capture":[85],"thespatial":[86],"dependencywithin":[87],"internal":[88],"representations":[89],"of":[90,199,205],"images.":[92],"To":[93],"overcome":[94],"these":[95],"problems":[96],"above,":[97],"we":[98],"decompose":[99],"generation":[103],"task":[104],"into":[105],"two":[106],"steps:":[107],"mouth":[108,124,148],"landmarks":[109,125],"prediction":[110],"and":[111,131,192,202],"synthesis.":[113],"First,":[114],"multimodal":[116],"method":[118],"multimedia":[127],"inputs":[128],"(both":[129],"audio).":[132],"Second,":[133],"named":[136],"Face2Vid":[137],"frames":[143,188],"conditioned":[144],"on":[145,196],"predicted":[147],"landmarks.":[149],"In":[150],"Face2Vid,":[151],"optical":[153],"flow":[154],"employed":[156],"model":[158,171],"temporal":[160],"dependency":[161,174],"between":[162],"frames,":[163],"meanwhile,":[164],"self-attention":[166],"mechanism":[167],"introduced":[169],"spatial":[173],"across":[175],"regions.":[177],"Extensive":[178],"experiments":[179],"demonstrate":[180],"that":[181],"our":[182],"approach":[183],"can":[184],"photo-realistic":[186],"background,":[191],"exhibit":[193],"superiorities":[195],"synchronization":[198],"movements":[201],"smooth":[203],"transition":[204],"facial":[206],"movements.":[207]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":15},{"year":2023,"cited_by_count":11},{"year":2022,"cited_by_count":11},{"year":2021,"cited_by_count":9},{"year":2020,"cited_by_count":2}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-10-10T00:00:00"}