{"id":"https://openalex.org/W4206517532","doi":"https://doi.org/10.1109/tmm.2022.3142387","title":"Audio-Driven Talking Face Video Generation With Dynamic Convolution Kernels","display_name":"Audio-Driven Talking Face Video Generation With Dynamic Convolution Kernels","publication_year":2022,"publication_date":"2022-01-13","ids":{"openalex":"https://openalex.org/W4206517532","doi":"https://doi.org/10.1109/tmm.2022.3142387"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2022.3142387","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2022.3142387","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2201.05986","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101402532","display_name":"Zipeng Ye","orcid":"https://orcid.org/0000-0002-4322-7550"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zipeng Ye","raw_affiliation_strings":["MOE-Key Laboratory of Pervasive Computing, BNRist, the Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-4322-7550","affiliations":[{"raw_affiliation_string":"MOE-Key Laboratory of Pervasive Computing, BNRist, the Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030085407","display_name":"Mengfei Xia","orcid":"https://orcid.org/0000-0002-8765-815X"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mengfei Xia","raw_affiliation_strings":["MOE-Key Laboratory of Pervasive Computing, BNRist, the Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MOE-Key Laboratory of Pervasive Computing, BNRist, the Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054879013","display_name":"Ran Yi","orcid":"https://orcid.org/0000-0003-1858-3358"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ran Yi","raw_affiliation_strings":["Department of Computer Science and Engineering, Shanghai Jiao Tong University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1858-3358","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Shanghai Jiao Tong University, Beijing, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101904821","display_name":"Juyong Zhang","orcid":"https://orcid.org/0000-0002-1805-1426"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Juyong Zhang","raw_affiliation_strings":["School of Mathematical Sciences, University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0000-0002-1805-1426","affiliations":[{"raw_affiliation_string":"School of Mathematical Sciences, University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067850699","display_name":"Yu\u2010Kun Lai","orcid":"https://orcid.org/0000-0002-2094-5680"},"institutions":[{"id":"https://openalex.org/I79510175","display_name":"Cardiff University","ror":"https://ror.org/03kk7td41","country_code":"GB","type":"education","lineage":["https://openalex.org/I79510175"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Yu-Kun Lai","raw_affiliation_strings":["School of Computer Science and Informatics, Cardiff University, Cardiff, U.K"],"raw_orcid":"https://orcid.org/0000-0002-2094-5680","affiliations":[{"raw_affiliation_string":"School of Computer Science and Informatics, Cardiff University, Cardiff, U.K","institution_ids":["https://openalex.org/I79510175"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084155598","display_name":"Xuwei Huang","orcid":"https://orcid.org/0000-0003-4222-3758"},"institutions":[{"id":"https://openalex.org/I2801745840","display_name":"Kwai Chung Hospital","ror":"https://ror.org/05kz7bw59","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I1294586568","https://openalex.org/I2801745840"]},{"id":"https://openalex.org/I4401726859","display_name":"Kuaishou (China)","ror":"https://ror.org/0258as409","country_code":null,"type":"company","lineage":["https://openalex.org/I4401726859"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuwei Huang","raw_affiliation_strings":["Kwai Inc, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kwai Inc, Beijing, China","institution_ids":["https://openalex.org/I2801745840","https://openalex.org/I4401726859"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100730836","display_name":"Guoxin Zhang","orcid":"https://orcid.org/0000-0002-7531-0404"},"institutions":[{"id":"https://openalex.org/I2801745840","display_name":"Kwai Chung Hospital","ror":"https://ror.org/05kz7bw59","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I1294586568","https://openalex.org/I2801745840"]},{"id":"https://openalex.org/I4401726859","display_name":"Kuaishou (China)","ror":"https://ror.org/0258as409","country_code":null,"type":"company","lineage":["https://openalex.org/I4401726859"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guoxin Zhang","raw_affiliation_strings":["Kwai Inc, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kwai Inc, Beijing, China","institution_ids":["https://openalex.org/I2801745840","https://openalex.org/I4401726859"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008076279","display_name":"Yong\u2010Jin Liu","orcid":"https://orcid.org/0000-0001-5774-1916"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yong-Jin Liu","raw_affiliation_strings":["MOE-Key Laboratory of Pervasive Computing, BNRist, the Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-5774-1916","affiliations":[{"raw_affiliation_string":"MOE-Key Laboratory of Pervasive Computing, BNRist, the Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.0409,"has_fulltext":true,"cited_by_count":53,"citation_normalized_percentile":{"value":0.97460159,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"25","issue":null,"first_page":"2033","last_page":"2046"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8943278789520264},{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.6776558756828308},{"id":"https://openalex.org/keywords/convolution","display_name":"Convolution (computer science)","score":0.6587095260620117},{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.5810718536376953},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5560380220413208},{"id":"https://openalex.org/keywords/face","display_name":"Face (sociological concept)","score":0.5407386422157288},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.49338141083717346},{"id":"https://openalex.org/keywords/video-processing","display_name":"Video processing","score":0.47537532448768616},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4336739182472229},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.22591739892959595}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8943278789520264},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.6776558756828308},{"id":"https://openalex.org/C45347329","wikidata":"https://www.wikidata.org/wiki/Q5166604","display_name":"Convolution (computer science)","level":3,"score":0.6587095260620117},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.5810718536376953},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5560380220413208},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.5407386422157288},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.49338141083717346},{"id":"https://openalex.org/C65483669","wikidata":"https://www.wikidata.org/wiki/Q3536669","display_name":"Video processing","level":2,"score":0.47537532448768616},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4336739182472229},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.22591739892959595},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/tmm.2022.3142387","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2022.3142387","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2201.05986","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2201.05986","pdf_url":"https://arxiv.org/pdf/2201.05986","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:https://orca.cardiff.ac.uk:146847","is_oa":true,"landing_page_url":null,"pdf_url":"https://orca.cardiff.ac.uk/id/eprint/146847/1/DynConvKernelsTMM.pdf","source":{"id":"https://openalex.org/S4306401195","display_name":"ORCA Online Research @Cardiff (Cardiff University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I79510175","host_organization_name":"Cardiff University","host_organization_lineage":["https://openalex.org/I79510175"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"pmh:oai:http://orca-dev.cardiff.ac.uk:146847","is_oa":false,"landing_page_url":"https://orca.cardiff.ac.uk/id/eprint/146847/","pdf_url":null,"source":{"id":"https://openalex.org/S4306401195","display_name":"ORCA Online Research @Cardiff (Cardiff University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I79510175","host_organization_name":"Cardiff University","host_organization_lineage":["https://openalex.org/I79510175"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2201.05986","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2201.05986","pdf_url":"https://arxiv.org/pdf/2201.05986","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1736198338","display_name":null,"funder_award_id":"61725204","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8721642152","display_name":null,"funder_award_id":"2021SHZDZX0102","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322392","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":59,"referenced_works":["https://openalex.org/W1569907127","https://openalex.org/W1686810756","https://openalex.org/W1901129140","https://openalex.org/W2017107803","https://openalex.org/W2045831935","https://openalex.org/W2053101950","https://openalex.org/W2105934661","https://openalex.org/W2115252128","https://openalex.org/W2158449659","https://openalex.org/W2237250383","https://openalex.org/W2486034530","https://openalex.org/W2546919788","https://openalex.org/W2584992898","https://openalex.org/W2593414223","https://openalex.org/W2594690981","https://openalex.org/W2619383789","https://openalex.org/W2770534877","https://openalex.org/W2789917104","https://openalex.org/W2799058067","https://openalex.org/W2806833697","https://openalex.org/W2883861033","https://openalex.org/W2884460600","https://openalex.org/W2899771611","https://openalex.org/W2944294033","https://openalex.org/W2960274051","https://openalex.org/W2962785568","https://openalex.org/W2962793481","https://openalex.org/W2963081548","https://openalex.org/W2963290645","https://openalex.org/W2964045146","https://openalex.org/W2964095416","https://openalex.org/W2964559396","https://openalex.org/W2967294981","https://openalex.org/W2967510410","https://openalex.org/W2979894294","https://openalex.org/W2981609494","https://openalex.org/W2990452356","https://openalex.org/W2991833656","https://openalex.org/W2995879646","https://openalex.org/W3008734941","https://openalex.org/W3034257218","https://openalex.org/W3034421924","https://openalex.org/W3034588700","https://openalex.org/W3081492798","https://openalex.org/W3087121792","https://openalex.org/W3097792222","https://openalex.org/W3101631197","https://openalex.org/W3105604940","https://openalex.org/W6637373629","https://openalex.org/W6639824700","https://openalex.org/W6677618333","https://openalex.org/W6737896281","https://openalex.org/W6751750676","https://openalex.org/W6752944736","https://openalex.org/W6756040250","https://openalex.org/W6766674170","https://openalex.org/W6769310319","https://openalex.org/W6774337803","https://openalex.org/W6864424756"],"related_works":["https://openalex.org/W4321487865","https://openalex.org/W4313906399","https://openalex.org/W2964954556","https://openalex.org/W3034421924","https://openalex.org/W4386858688","https://openalex.org/W2982536526","https://openalex.org/W4380302312","https://openalex.org/W3008689640","https://openalex.org/W4385338604","https://openalex.org/W3081626085"],"abstract_inverted_index":{"In":[0],"this":[1],"paper,":[2],"we":[3],"present":[4],"a":[5,16,69,78],"dynamic":[6],"convolution":[7],"kernel":[8],"(DCK)":[9],"strategy":[10],"for":[11,61],"convolutional":[12,18],"neural":[13],"networks.":[14],"Using":[15],"fully":[17],"network":[19],"with":[20,97],"the":[21,109,113],"proposed":[22,56],"DCKs,":[23],"high-quality":[24,94],"talking-face":[25,95],"video":[26,65,96],"can":[27,92],"be":[28],"generated":[29],"from":[30],"multi-modal":[31],"sources":[32],"(i.e.,":[33],"unmatched":[34],"audio":[35],"and":[36,41,52,103,108],"video)":[37],"in":[38],"real":[39],"time,":[40],"our":[42,90,106,116],"trained":[43],"model":[44],"is":[45],"robust":[46],"to":[47,68,81],"different":[48],"identities,":[49],"head":[50],"postures,":[51],"input":[53],"audios.":[54],"Our":[55],"DCKs":[57,84],"are":[58],"specially":[59],"designed":[60],"audio-driven":[62],"talking":[63],"face":[64],"generation,":[66],"leading":[67],"simple":[70],"yet":[71],"effective":[72],"end-to-end":[73],"system.":[74],"We":[75],"also":[76],"provide":[77],"theoretical":[79],"analysis":[80],"interpret":[82],"why":[83],"work.":[85],"Experimental":[86],"results":[87],"show":[88],"that":[89],"method":[91,107],"generate":[93],"background":[98],"at":[99],"60":[100],"fps.":[101],"Comparison":[102],"evaluation":[104],"between":[105],"state-of-the-art":[110],"methods":[111],"demonstrate":[112],"superiority":[114],"of":[115],"method.":[117]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":20},{"year":2023,"cited_by_count":17},{"year":2022,"cited_by_count":2}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2022-01-25T00:00:00"}