{"id":"https://openalex.org/W4416123565","doi":"https://doi.org/10.1109/iccv51701.2025.01166","title":"Audio-Visual Controlled Video Diffusion with Masked Selective State Spaces Modeling for Natural Talking Head Generation","display_name":"Audio-Visual Controlled Video Diffusion with Masked Selective State Spaces Modeling for Natural Talking Head Generation","publication_year":2025,"publication_date":"2025-10-19","ids":{"openalex":"https://openalex.org/W4416123565","doi":"https://doi.org/10.1109/iccv51701.2025.01166"},"language":"en","primary_location":{"id":"doi:10.1109/iccv51701.2025.01166","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv51701.2025.01166","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2504.02542","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074327829","display_name":"Fa-Ting Hong","orcid":null},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Fa-Ting Hong","raw_affiliation_strings":["HKUST"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"HKUST","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100628169","display_name":"Zhiwei Xu","orcid":"https://orcid.org/0000-0002-1480-7265"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zunnan Xu","raw_affiliation_strings":["Tencent"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zixiang Zhou","orcid":null},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zixiang Zhou","raw_affiliation_strings":["Tencent"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045140292","display_name":"Jun Zhou","orcid":"https://orcid.org/0000-0001-6033-6102"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Zhou","raw_affiliation_strings":["Tencent"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104326372","display_name":"Xiu Li","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiu Li","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108753476","display_name":"Qin Lin","orcid":null},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qin Lin","raw_affiliation_strings":["Tencent"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011375148","display_name":"Qinglin Lu","orcid":"https://orcid.org/0000-0002-4584-0826"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qinglin Lu","raw_affiliation_strings":["Tencent"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100341930","display_name":"Dan Xu","orcid":"https://orcid.org/0000-0002-4336-9700"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Dan Xu","raw_affiliation_strings":["HKUST"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"HKUST","institution_ids":["https://openalex.org/I200769079"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.38051966,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"12549","last_page":"12558"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.7609999775886536,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.7609999775886536,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.21220000088214874,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.0035000001080334187,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.6086000204086304},{"id":"https://openalex.org/keywords/signal","display_name":"SIGNAL (programming language)","score":0.6032999753952026},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.5152000188827515},{"id":"https://openalex.org/keywords/control-signal","display_name":"Control signal","score":0.4950000047683716},{"id":"https://openalex.org/keywords/head","display_name":"Head (geology)","score":0.44920000433921814},{"id":"https://openalex.org/keywords/face","display_name":"Face (sociological concept)","score":0.4296000003814697},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.429500013589859},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.34549999237060547}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7609999775886536},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.6086000204086304},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.6032999753952026},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.5152000188827515},{"id":"https://openalex.org/C3018134525","wikidata":"https://www.wikidata.org/wiki/Q2501541","display_name":"Control signal","level":3,"score":0.4950000047683716},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.47369998693466187},{"id":"https://openalex.org/C2780312720","wikidata":"https://www.wikidata.org/wiki/Q5689100","display_name":"Head (geology)","level":2,"score":0.44920000433921814},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44269999861717224},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.4296000003814697},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.429500013589859},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.34549999237060547},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3285999894142151},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.321399986743927},{"id":"https://openalex.org/C17500928","wikidata":"https://www.wikidata.org/wiki/Q959968","display_name":"Control system","level":2,"score":0.2978000044822693},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2955999970436096},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.28450000286102295},{"id":"https://openalex.org/C104267543","wikidata":"https://www.wikidata.org/wiki/Q208163","display_name":"Signal processing","level":3,"score":0.28279998898506165},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.2628999948501587},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.2563999891281128},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.25369998812675476},{"id":"https://openalex.org/C65483669","wikidata":"https://www.wikidata.org/wiki/Q3536669","display_name":"Video processing","level":2,"score":0.25290000438690186},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2524000108242035}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/iccv51701.2025.01166","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv51701.2025.01166","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2504.02542","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.02542","pdf_url":"https://arxiv.org/pdf/2504.02542","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-149524","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-149524","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference paper"},{"id":"doi:10.48550/arxiv.2504.02542","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2504.02542","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2504.02542","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.02542","pdf_url":"https://arxiv.org/pdf/2504.02542","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416123565.pdf","grobid_xml":"https://content.openalex.org/works/W4416123565.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Talking":[0],"head":[1,51],"synthesis":[2],"is":[3,80],"vital":[4],"for":[5,49],"virtual":[6],"avatars":[7],"and":[8,46,101,161],"human-computer":[9],"interaction.":[10],"However,":[11],"most":[12],"existing":[13],"methods":[14],"are":[15],"typically":[16],"limited":[17],"to":[18,72,112,133],"accepting":[19],"control":[20,45,48,73,87,135,145],"from":[21],"a":[22,59,68,125],"single":[23],"primary":[24],"modality,":[25],"restricting":[26],"their":[27],"practical":[28],"utility.":[29],"To":[30,91],"this":[31],"end,":[32],"we":[33,57,103,123],"introduce":[34,124],"\\textbf{ACTalker},":[35],"an":[36],"end-to-end":[37],"video":[38,52,89,98],"diffusion":[39],"framework":[40],"that":[41,128,150,162],"supports":[42],"both":[43,99,117],"multi-signals":[44],"single-signal":[47],"talking":[50],"generation.":[53,90],"For":[54],"multiple":[55,64,168],"control,":[56],"design":[58],"parallel":[60],"mamba":[61,106,142,164],"structure":[62],"with":[63],"branches,":[65,84],"each":[66,120,130],"utilizing":[67],"separate":[69],"driving":[70,110,131,169],"signal":[71,132],"specific":[74],"facial":[75,138,155],"regions.":[76],"A":[77],"gate":[78],"mechanism":[79],"applied":[81],"across":[82,116],"all":[83],"providing":[85],"flexible":[86],"over":[88],"ensure":[92],"natural":[93],"coordination":[94],"of":[95],"the":[96,105,141,163],"controlled":[97],"temporally":[100],"spatially,":[102],"employ":[104],"structure,":[107,143],"which":[108],"enables":[109],"signals":[111,160],"manipulate":[113],"feature":[114],"tokens":[115],"dimensions":[118],"in":[119],"branch.":[121],"Additionally,":[122],"mask-drop":[126],"strategy":[127],"allows":[129],"independently":[134],"its":[136],"corresponding":[137],"region":[139],"within":[140],"preventing":[144],"conflicts.":[146],"Experimental":[147],"results":[148],"demonstrate":[149],"our":[151],"method":[152],"produces":[153],"natural-looking":[154],"videos":[156],"driven":[157],"by":[158],"diverse":[159],"layer":[165],"seamlessly":[166],"integrates":[167],"modalities":[170],"without":[171],"conflict.":[172],"The":[173],"project":[174],"website":[175],"can":[176],"be":[177],"found":[178],"at":[179],"https://harlanhong.github.io/publications/actalker/index.html.":[180]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}