{"id":"https://openalex.org/W4393399665","doi":"https://doi.org/10.48550/arxiv.2403.20153","title":"Talk3D: High-Fidelity Talking Portrait Synthesis via Personalized 3D Generative Prior","display_name":"Talk3D: High-Fidelity Talking Portrait Synthesis via Personalized 3D Generative Prior","publication_year":2024,"publication_date":"2024-03-29","ids":{"openalex":"https://openalex.org/W4393399665","doi":"https://doi.org/10.48550/arxiv.2403.20153"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2403.20153","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.20153","pdf_url":"https://arxiv.org/pdf/2403.20153","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2403.20153","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Ko, Jaehoon","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I197347611","display_name":"Korea University","ror":"https://ror.org/047dqcg40","country_code":"KR","type":"education","lineage":["https://openalex.org/I197347611"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Ko, Jaehoon","raw_affiliation_strings":["Korea University , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Korea University , Republic of Korea","institution_ids":["https://openalex.org/I197347611"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025337397","display_name":"Kyusun Cho","orcid":"https://orcid.org/0000-0002-0177-0694","observed_orcids":["https://orcid.org/0000-0002-0177-0694"]},"institutions":[{"id":"https://openalex.org/I197347611","display_name":"Korea University","ror":"https://ror.org/047dqcg40","country_code":"KR","type":"education","lineage":["https://openalex.org/I197347611"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Cho, Kyusun","raw_affiliation_strings":["Korea University , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Korea University , Republic of Korea","institution_ids":["https://openalex.org/I197347611"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008535930","display_name":"Joungbin Lee","orcid":"https://orcid.org/0009-0008-1707-9727","observed_orcids":["https://orcid.org/0009-0008-1707-9727"]},"institutions":[{"id":"https://openalex.org/I197347611","display_name":"Korea University","ror":"https://ror.org/047dqcg40","country_code":"KR","type":"education","lineage":["https://openalex.org/I197347611"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Lee, Joungbin","raw_affiliation_strings":["Korea University , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Korea University , Republic of Korea","institution_ids":["https://openalex.org/I197347611"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102638300","display_name":"Heeji Yoon","orcid":"https://orcid.org/0009-0004-9238-321X","observed_orcids":["https://orcid.org/0009-0004-9238-321X"]},"institutions":[{"id":"https://openalex.org/I197347611","display_name":"Korea University","ror":"https://ror.org/047dqcg40","country_code":"KR","type":"education","lineage":["https://openalex.org/I197347611"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Yoon, Heeji","raw_affiliation_strings":["Korea University , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Korea University , Republic of Korea","institution_ids":["https://openalex.org/I197347611"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100342111","display_name":"Sangmin Lee","orcid":"https://orcid.org/0000-0002-5215-2546","observed_orcids":["https://orcid.org/0000-0002-5215-2546"]},"institutions":[{"id":"https://openalex.org/I197347611","display_name":"Korea University","ror":"https://ror.org/047dqcg40","country_code":"KR","type":"education","lineage":["https://openalex.org/I197347611"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Lee, Sangmin","raw_affiliation_strings":["Korea University , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Korea University , Republic of Korea","institution_ids":["https://openalex.org/I197347611"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113842642","display_name":"Sangjun Ahn","orcid":"https://orcid.org/0000-0002-2098-0718","observed_orcids":["https://orcid.org/0000-0002-2098-0718"]},"institutions":[{"id":"https://openalex.org/I4210135449","display_name":"NCSOFT (South Korea)","ror":"https://ror.org/03q4mza74","country_code":"KR","type":"company","lineage":["https://openalex.org/I4210135449"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Ahn, Sangjun","raw_affiliation_strings":["NCSOFT , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NCSOFT , Republic of Korea","institution_ids":["https://openalex.org/I4210135449"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5085363061","display_name":"Seungryong Kim","orcid":"https://orcid.org/0000-0003-2927-6273","observed_orcids":["https://orcid.org/0000-0003-2927-6273"]},"institutions":[{"id":"https://openalex.org/I197347611","display_name":"Korea University","ror":"https://ror.org/047dqcg40","country_code":"KR","type":"education","lineage":["https://openalex.org/I197347611"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Kim, Seungryong","raw_affiliation_strings":["Korea University , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Korea University , Republic of Korea","institution_ids":["https://openalex.org/I197347611"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.7645999789237976,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.7645999789237976,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.13439999520778656,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10719","display_name":"3D Shape Modeling and Analysis","score":0.06920000165700912,"subfield":{"id":"https://openalex.org/subfields/1704","display_name":"Computer Graphics and Computer-Aided Design"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/talking-head-synthesis","display_name":"talking head synthesis","score":0.8970000147819519},{"id":"https://openalex.org/keywords/audio-driven-face-animation","display_name":"audio-driven face animation","score":0.7400000095367432},{"id":"https://openalex.org/keywords/neural-radiance-fields","display_name":"neural radiance fields","score":0.9150000214576721}],"concepts":[{"id":"https://openalex.org/C162462552","wikidata":"https://www.wikidata.org/wiki/Q134307","display_name":"Portrait","level":2,"score":0.8673697710037231},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.6443811655044556},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.6260270476341248},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.5408767461776733},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4939196705818176},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3444957733154297},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.333219438791275},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.2671511769294739},{"id":"https://openalex.org/C52119013","wikidata":"https://www.wikidata.org/wiki/Q50637","display_name":"Art history","level":1,"score":0.22425231337547302},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.053623735904693604}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2403.20153","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.20153","pdf_url":"https://arxiv.org/pdf/2403.20153","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2403.20153","doi":"https://doi.org/10.48550/arxiv.2403.20153","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2403.20153","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2403.20153","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.20153","pdf_url":"https://arxiv.org/pdf/2403.20153","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4393399665.pdf","grobid_xml":"https://content.openalex.org/works/W4393399665.grobid-xml"},"referenced_works_count":7,"referenced_works":["https://openalex.org/W4226204502","https://openalex.org/W4312325284","https://openalex.org/W4367701091","https://openalex.org/W4377131125","https://openalex.org/W4390872116","https://openalex.org/W4402728159","https://openalex.org/W4411049949"],"related_works":["https://openalex.org/W4396822349","https://openalex.org/W4408352166","https://openalex.org/W7125899129","https://openalex.org/W7213424828","https://openalex.org/W4413557526","https://openalex.org/W3112970464","https://openalex.org/W4392911120","https://openalex.org/W4415183939","https://openalex.org/W4386133969","https://openalex.org/W4415428508"],"abstract_inverted_index":{"Recent":[0],"methods":[1],"for":[2],"audio-driven":[3,58],"talking":[4,16,59],"head":[5,60,141],"synthesis":[6,61],"often":[7,31],"optimize":[8],"neural":[9],"radiance":[10],"fields":[11],"(NeRF)":[12],"on":[13],"a":[14,56,89],"monocular":[15,49],"portrait":[17],"video,":[18],"leveraging":[19],"its":[20,69],"capability":[21],"to":[22,33,39,123,127],"render":[23],"high-fidelity":[24],"and":[25,159],"3D-consistent":[26],"novel-view":[27],"frames.":[28],"However,":[29],"they":[30],"struggle":[32],"reconstruct":[34,68],"complete":[35],"face":[36,99],"geometry":[37],"due":[38],"the":[40,47,76,82,97,102],"absence":[41],"of":[42,156],"comprehensive":[43],"3D":[44,84],"information":[45],"in":[46,101,133,154],"input":[48],"videos.":[50],"In":[51],"this":[52],"paper,":[53],"we":[54,87],"introduce":[55],"novel":[57,90],"framework,":[62],"called":[63],"Talk3D,":[64],"that":[65,95],"can":[66],"faithfully":[67],"plausible":[70],"facial":[71,136],"geometries":[72,137],"by":[73,106,114],"effectively":[74,119],"adopting":[75],"pre-trained":[77],"3D-aware":[78],"generative":[79,85],"prior.":[80],"Given":[81],"personalized":[83],"model,":[86],"present":[88],"audio-guided":[91],"attention":[92],"U-Net":[93],"architecture":[94],"predicts":[96],"dynamic":[98],"variations":[100,121],"NeRF":[103],"space":[104],"driven":[105],"audio.":[107],"Furthermore,":[108],"our":[109,130,149],"model":[110],"is":[111],"further":[112],"modulated":[113],"audio-unrelated":[115],"conditioning":[116],"tokens":[117],"which":[118],"disentangle":[120],"unrelated":[122],"audio":[124],"features.":[125],"Compared":[126],"existing":[128],"methods,":[129],"method":[131],"excels":[132],"generating":[134],"realistic":[135],"even":[138],"under":[139],"extreme":[140],"poses.":[142],"We":[143],"also":[144],"conduct":[145],"extensive":[146],"experiments":[147],"showing":[148],"approach":[150],"surpasses":[151],"state-of-the-art":[152],"benchmarks":[153],"terms":[155],"both":[157],"quantitative":[158],"qualitative":[160],"evaluations.":[161]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-04-02T00:00:00"}