{"id":"https://openalex.org/W4317438017","doi":"https://doi.org/10.48550/arxiv.2301.06059","title":"Learning Audio-Driven Viseme Dynamics for 3D Face Animation","display_name":"Learning Audio-Driven Viseme Dynamics for 3D Face Animation","publication_year":2023,"publication_date":"2023-01-15","ids":{"openalex":"https://openalex.org/W4317438017","doi":"https://doi.org/10.48550/arxiv.2301.06059"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2301.06059","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2301.06059","pdf_url":"https://arxiv.org/pdf/2301.06059","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2301.06059","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5079937813","display_name":"Linchao Bao","orcid":"https://orcid.org/0000-0001-9543-3754","observed_orcids":["https://orcid.org/0000-0001-9543-3754"]},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bao, Linchao","raw_affiliation_strings":["Tencent AI Lab , China","Tencent AI Lab , Shenzhen , China,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent AI Lab , China","institution_ids":["https://openalex.org/I2250653659"]},{"raw_affiliation_string":"Tencent AI Lab , Shenzhen , China,","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066359113","display_name":"Haoxian Zhang","orcid":"https://orcid.org/0000-0001-7078-868X","observed_orcids":["https://orcid.org/0000-0001-7078-868X"]},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhang, Haoxian","raw_affiliation_strings":["Tencent AI Lab , China","Tencent AI Lab , Shenzhen , China;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent AI Lab , China","institution_ids":["https://openalex.org/I2250653659"]},{"raw_affiliation_string":"Tencent AI Lab , Shenzhen , China;","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100298506","display_name":"Yue Qian","orcid":"https://orcid.org/0000-0001-5003-2885","observed_orcids":["https://orcid.org/0000-0001-5003-2885","https://orcid.org/0009-0006-7910-9560","https://orcid.org/0000-0003-2120-5403","https://orcid.org/0000-0001-6253-3521"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qian, Yue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043245225","display_name":"Tangli Xue","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xue, Tangli","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012644973","display_name":"Changhai Chen","orcid":"https://orcid.org/0000-0001-5014-2574","observed_orcids":["https://orcid.org/0000-0001-5014-2574"]},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chen, Changhai","raw_affiliation_strings":["Tencent AI Lab , Shenzhen , China;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent AI Lab , Shenzhen , China;","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044236581","display_name":"Xuefei Zhe","orcid":"https://orcid.org/0000-0002-5005-7166","observed_orcids":["https://orcid.org/0000-0002-5005-7166"]},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhe, Xuefei","raw_affiliation_strings":["Tencent AI Lab , Shenzhen , China; Di Kang , Tencent AI Lab , Shenzhen , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent AI Lab , Shenzhen , China; Di Kang , Tencent AI Lab , Shenzhen , China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"last","author":{"id":null,"display_name":"Kang, Di","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kang, Di","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":20,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12290","display_name":"Human Motion and Animation","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1704","display_name":"Computer Graphics and Computer-Aided Design"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12290","display_name":"Human Motion and Animation","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1704","display_name":"Computer Graphics and Computer-Aided Design"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.0032999999821186066,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.003100000089034438,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/3d-face-animation","display_name":"3D face animation","score":0.8529999852180481},{"id":"https://openalex.org/keywords/lip-synchronization","display_name":"lip synchronization","score":0.8190000057220459},{"id":"https://openalex.org/keywords/multilingual-speech","display_name":"multilingual speech","score":0.8370000123977661}],"concepts":[{"id":"https://openalex.org/C33767174","wikidata":"https://www.wikidata.org/wiki/Q371190","display_name":"Viseme","level":4,"score":0.97922682762146},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7765297889709473},{"id":"https://openalex.org/C502989409","wikidata":"https://www.wikidata.org/wiki/Q11425","display_name":"Animation","level":2,"score":0.6884686350822449},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5316149592399597},{"id":"https://openalex.org/C138591656","wikidata":"https://www.wikidata.org/wiki/Q5157538","display_name":"Computer facial animation","level":4,"score":0.5224403738975525},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.495524525642395},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.4893021285533905},{"id":"https://openalex.org/C69369342","wikidata":"https://www.wikidata.org/wiki/Q1401416","display_name":"Computer animation","level":3,"score":0.4876073896884918},{"id":"https://openalex.org/C14999030","wikidata":"https://www.wikidata.org/wiki/Q16346","display_name":"Speech synthesis","level":2,"score":0.47491204738616943},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.351370632648468},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.21006327867507935},{"id":"https://openalex.org/C504749915","wikidata":"https://www.wikidata.org/wiki/Q9010971","display_name":"Speech technology","level":3,"score":0.08746403455734253},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2301.06059","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2301.06059","pdf_url":"https://arxiv.org/pdf/2301.06059","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2301.06059","doi":"https://doi.org/10.48550/arxiv.2301.06059","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2301.06059","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2301.06059","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2301.06059","pdf_url":"https://arxiv.org/pdf/2301.06059","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4317438017.pdf","grobid_xml":"https://content.openalex.org/works/W4317438017.grobid-xml"},"referenced_works_count":5,"referenced_works":["https://openalex.org/W788203010","https://openalex.org/W2936774411","https://openalex.org/W2964121744","https://openalex.org/W3036914231","https://openalex.org/W3037057938"],"related_works":["https://openalex.org/W2950248412","https://openalex.org/W2804619907","https://openalex.org/W2142233983","https://openalex.org/W4415433159","https://openalex.org/W2244377763","https://openalex.org/W2138147384","https://openalex.org/W2550297256","https://openalex.org/W2095156353","https://openalex.org/W7095337593","https://openalex.org/W2047908040"],"abstract_inverted_index":{"We":[0,149],"present":[1],"a":[2,43,133],"novel":[3,44],"audio-driven":[4],"facial":[5,14,172],"animation":[6,147,185],"approach":[7,21,41,136,175],"that":[8,49,151],"can":[9,69,156,179],"generate":[10],"realistic":[11,169],"lip-synchronized":[12],"3D":[13],"animations":[15,167],"from":[16,25,57,107,124],"the":[17,61,65,105,120,152],"input":[18,121],"audio.":[19],"Our":[20,112,174],"learns":[22],"viseme":[23,30,46,55,67,110,134,140,154],"dynamics":[24],"speech":[26,35,58,84,146],"videos,":[27],"produces":[28],"animator-friendly":[29],"curves,":[31],"and":[32,77,86,170,178],"supports":[33],"multilingual":[34,83],"inputs.":[36],"The":[37],"core":[38],"of":[39,63,94,126],"our":[40],"is":[42,142,176],"parametric":[45],"fitting":[47],"algorithm":[48],"utilizes":[50],"phoneme":[51],"priors":[52],"to":[53,79,88,103,109,159,163],"extract":[54],"parameters":[56],"videos.":[59],"With":[60],"guidance":[62],"phonemes,":[64,73],"extracted":[66],"curves":[68,155],"better":[70],"correlate":[71],"with":[72,168],"thus":[74],"more":[75],"controllable":[76],"friendly":[78],"animators.":[80],"To":[81],"support":[82],"inputs":[85],"generalizability":[87],"unseen":[89],"voices,":[90],"we":[91],"take":[92],"advantage":[93],"deep":[95],"audio":[96,108,122],"feature":[97],"models":[98],"pretrained":[99],"on":[100],"multiple":[101],"languages":[102],"learn":[104],"mapping":[106,114],"curves.":[111],"audio-to-curves":[113],"achieves":[115],"state-of-the-art":[116],"performance":[117],"even":[118],"when":[119],"suffers":[123],"distortions":[125],"volume,":[127],"pitch,":[128],"speed,":[129],"or":[130,190],"noise.":[131],"Lastly,":[132],"scanning":[135],"for":[137,144],"acquiring":[138],"high-fidelity":[139],"assets":[141],"presented":[143],"efficient":[145],"production.":[148],"show":[150],"predicted":[153],"be":[157,180],"applied":[158],"different":[160],"viseme-rigged":[161],"characters":[162],"yield":[164],"various":[165],"personalized":[166],"natural":[171],"motions.":[173],"artist-friendly":[177],"easily":[181],"integrated":[182],"into":[183],"typical":[184],"production":[186],"workflows":[187],"including":[188],"blendshape":[189],"bone":[191],"based":[192],"animation.":[193]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":4}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2023-01-19T00:00:00"}