{"id":"https://openalex.org/W4401483653","doi":"https://doi.org/10.1145/3673901","title":"Multi-Modal Driven Pose-Controllable Talking Head Generation","display_name":"Multi-Modal Driven Pose-Controllable Talking Head Generation","publication_year":2024,"publication_date":"2024-08-10","ids":{"openalex":"https://openalex.org/W4401483653","doi":"https://doi.org/10.1145/3673901"},"language":"en","primary_location":{"id":"doi:10.1145/3673901","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673901","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673901","source":{"id":"https://openalex.org/S19610489","display_name":"ACM Transactions on Multimedia Computing Communications and Applications","issn_l":"1551-6857","issn":["1551-6857","1551-6865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Multimedia Computing, Communications, and Applications","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3673901","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102641633","display_name":"Kuiyuan Sun","orcid":"https://orcid.org/0009-0007-0077-7388"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kuiyuan Sun","raw_affiliation_strings":["Institute of Information Science, Beijing Key Laboratory of Advanced Information Science and Network Technology, Beijing Jiaotong University, China"],"raw_orcid":"https://orcid.org/0009-0007-0077-7388","affiliations":[{"raw_affiliation_string":"Institute of Information Science, Beijing Key Laboratory of Advanced Information Science and Network Technology, Beijing Jiaotong University, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100393817","display_name":"Xiaolong Liu","orcid":"https://orcid.org/0000-0002-3940-207X"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaolong Liu","raw_affiliation_strings":["Institute of Information Science, Beijing Key Laboratory of Advanced Information Science and Network Technology, Beijing Jiaotong University, China"],"raw_orcid":"https://orcid.org/0000-0002-3940-207X","affiliations":[{"raw_affiliation_string":"Institute of Information Science, Beijing Key Laboratory of Advanced Information Science and Network Technology, Beijing Jiaotong University, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100784768","display_name":"Xiaolong Li","orcid":"https://orcid.org/0000-0001-6181-6044"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaolong Li","raw_affiliation_strings":["Institute of Information Science, Beijing Key Laboratory of Advanced Information Science and Network Technology, Beijing Jiaotong University, China"],"raw_orcid":"https://orcid.org/0000-0001-6181-6044","affiliations":[{"raw_affiliation_string":"Institute of Information Science, Beijing Key Laboratory of Advanced Information Science and Network Technology, Beijing Jiaotong University, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100362745","display_name":"Yao Zhao","orcid":"https://orcid.org/0000-0002-8581-9554"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yao Zhao","raw_affiliation_strings":["Institute of Information Science, Beijing Key Laboratory of Advanced Information Science and Network Technology, Beijing Jiaotong University, China"],"raw_orcid":"https://orcid.org/0000-0002-8581-9554","affiliations":[{"raw_affiliation_string":"Institute of Information Science, Beijing Key Laboratory of Advanced Information Science and Network Technology, Beijing Jiaotong University, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100392006","display_name":"Wei Wang","orcid":"https://orcid.org/0000-0002-5477-1017"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei wang","raw_affiliation_strings":["Institute of Information Science, Beijing Key Laboratory of Advanced Information Science and Network Technology, Beijing Jiaotong University, China"],"raw_orcid":"https://orcid.org/0000-0002-5477-1017","affiliations":[{"raw_affiliation_string":"Institute of Information Science, Beijing Key Laboratory of Advanced Information Science and Network Technology, Beijing Jiaotong University, China","institution_ids":["https://openalex.org/I21193070"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I21193070"],"apc_list":null,"apc_paid":null,"fwci":1.2262,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.80028248,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12290","display_name":"Human Motion and Animation","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8860999345779419},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.7307413816452026},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6620374917984009},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.6323191523551941},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.6216108798980713},{"id":"https://openalex.org/keywords/landmark","display_name":"Landmark","score":0.6081629991531372},{"id":"https://openalex.org/keywords/translation","display_name":"Translation (biology)","score":0.5504360198974609},{"id":"https://openalex.org/keywords/head","display_name":"Head (geology)","score":0.5307307839393616},{"id":"https://openalex.org/keywords/rotation","display_name":"Rotation (mathematics)","score":0.4494915008544922},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.44554463028907776}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8860999345779419},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.7307413816452026},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6620374917984009},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.6323191523551941},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.6216108798980713},{"id":"https://openalex.org/C2780297707","wikidata":"https://www.wikidata.org/wiki/Q4895393","display_name":"Landmark","level":2,"score":0.6081629991531372},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.5504360198974609},{"id":"https://openalex.org/C2780312720","wikidata":"https://www.wikidata.org/wiki/Q5689100","display_name":"Head (geology)","level":2,"score":0.5307307839393616},{"id":"https://openalex.org/C74050887","wikidata":"https://www.wikidata.org/wiki/Q848368","display_name":"Rotation (mathematics)","level":2,"score":0.4494915008544922},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.44554463028907776},{"id":"https://openalex.org/C105580179","wikidata":"https://www.wikidata.org/wiki/Q188928","display_name":"Messenger RNA","level":3,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C114793014","wikidata":"https://www.wikidata.org/wiki/Q52109","display_name":"Geomorphology","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3673901","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673901","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673901","source":{"id":"https://openalex.org/S19610489","display_name":"ACM Transactions on Multimedia Computing Communications and Applications","issn_l":"1551-6857","issn":["1551-6857","1551-6865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Multimedia Computing, Communications, and Applications","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3673901","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673901","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673901","source":{"id":"https://openalex.org/S19610489","display_name":"ACM Transactions on Multimedia Computing Communications and Applications","issn_l":"1551-6857","issn":["1551-6857","1551-6865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Multimedia Computing, Communications, and Applications","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.5600000023841858,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4401483653.pdf"},"referenced_works_count":32,"referenced_works":["https://openalex.org/W1901129140","https://openalex.org/W2107037917","https://openalex.org/W2237250383","https://openalex.org/W2331128040","https://openalex.org/W2603777577","https://openalex.org/W2738406145","https://openalex.org/W2946124159","https://openalex.org/W2962925415","https://openalex.org/W2962974533","https://openalex.org/W2963789946","https://openalex.org/W2964449965","https://openalex.org/W2981263323","https://openalex.org/W2990452356","https://openalex.org/W3019952993","https://openalex.org/W3034900344","https://openalex.org/W3081492798","https://openalex.org/W3101631197","https://openalex.org/W3109114891","https://openalex.org/W3112871073","https://openalex.org/W3154411171","https://openalex.org/W3174763799","https://openalex.org/W3180794345","https://openalex.org/W3186090335","https://openalex.org/W3204680331","https://openalex.org/W4200150166","https://openalex.org/W4200631136","https://openalex.org/W4205474609","https://openalex.org/W4214809420","https://openalex.org/W4286611278","https://openalex.org/W4291433229","https://openalex.org/W4293090828","https://openalex.org/W4312453532"],"related_works":["https://openalex.org/W2056853153","https://openalex.org/W2057559274","https://openalex.org/W2005087563","https://openalex.org/W2378111931","https://openalex.org/W2052388267","https://openalex.org/W2950647290","https://openalex.org/W1968481813","https://openalex.org/W2620829895","https://openalex.org/W2356918560","https://openalex.org/W627697492"],"abstract_inverted_index":{"Talking":[0],"head,":[1],"driving":[2,81],"a":[3,8,35,53,63,74,91,142,164,180],"source":[4,117,137,170],"image":[5,138,171],"to":[6,33,61,79,83,90,96,101,111,129,162,178],"generate":[7,62,97,179],"talking":[9,64,182],"video":[10],"using":[11],"other":[12],"modality":[13,37],"information,":[14],"has":[15],"made":[16],"great":[17],"progress":[18],"in":[19,199],"recent":[20],"years.":[21],"However,":[22],"there":[23],"are":[24,31,160],"two":[25],"main":[26],"issues:":[27],"1)":[28],"These":[29],"methods":[30,42],"designed":[32],"utilize":[34,58],"single":[36],"of":[38,195,201],"information.":[39],"2)":[40],"Most":[41],"cannot":[43],"control":[44,72,130],"head":[45,65,70,131,183,207],"pose.":[46],"To":[47],"address":[48],"these":[49],"problems,":[50],"we":[51],"propose":[52],"novel":[54],"framework":[55],"that":[56,189],"can":[57],"multi-modal":[59,86],"information":[60,82,87,114,123],"video,":[66],"while":[67],"achieving":[68,196],"arbitrary":[69],"pose":[71,132,208],"by":[73,141,146],"movement":[75],"sequence.":[76],"Specifically,":[77],"first,":[78],"extend":[80],"multiple":[84],"modalities,":[85],"is":[88,109,139,172,193],"encoded":[89],"unified":[92],"semantic":[93],"latent":[94],"space":[95],"expression":[98,148],"parameters.":[99],"Secondly,":[100],"disentangle":[102],"attributes,":[103],"the":[104,116,125,136,147,157,168,175],"3D":[105],"Morphable":[106],"Model":[107],"(3DMM)":[108],"utilized":[110,161],"obtain":[112],"identity":[113],"from":[115,124],"image,":[118],"and":[119,121,133,152,167,206],"translation":[120,150],"rotation":[122],"target":[126],"image.":[127],"Thirdly,":[128],"mouth":[134],"shape,":[135],"warped":[140,169],"motion":[143],"field":[144],"generated":[145],"parameter,":[149,151],"angle":[153],"parameter.":[154],"Finally,":[155],"all":[156],"above":[158],"parameters":[159],"render":[163],"landmark":[165,176],"map,":[166],"combined":[173],"with":[174],"map":[177],"delicate":[181],"video.":[184],"Our":[185],"experimental":[186],"results":[187],"demonstrate":[188],"our":[190],"proposed":[191],"method":[192],"capable":[194],"state-of-the-art":[197],"performance":[198],"terms":[200],"visual":[202],"quality,":[203],"lip-audio":[204],"synchronization,":[205],"control.":[209]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":3}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}