{"id":"https://openalex.org/W4392904562","doi":"https://doi.org/10.1109/icassp48485.2024.10447978","title":"FreeTalker: Controllable Speech and Text-Driven Gesture Generation Based on Diffusion Models for Enhanced Speaker Naturalness","display_name":"FreeTalker: Controllable Speech and Text-Driven Gesture Generation Based on Diffusion Models for Enhanced Speaker Naturalness","publication_year":2024,"publication_date":"2024-03-18","ids":{"openalex":"https://openalex.org/W4392904562","doi":"https://doi.org/10.1109/icassp48485.2024.10447978"},"language":"en","primary_location":{"id":"doi:10.1109/icassp48485.2024.10447978","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10447978","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113092484","display_name":"Sicheng Yang","orcid":"https://orcid.org/0009-0004-8472-9022"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]},{"id":"https://openalex.org/I4210114105","display_name":"Tsinghua\u2013Berkeley Shenzhen Institute","ror":"https://ror.org/02hhwwz98","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210114105","https://openalex.org/I95457486","https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Sicheng Yang","raw_affiliation_strings":["Tsinghua University,Tsinghua Shenzhen International Graduate School","Tsinghua Shenzhen International Graduate School, Tsinghua University"],"affiliations":[{"raw_affiliation_string":"Tsinghua University,Tsinghua Shenzhen International Graduate School","institution_ids":["https://openalex.org/I4210114105","https://openalex.org/I99065089"]},{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School, Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113145225","display_name":"Zunnan Xu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210114105","display_name":"Tsinghua\u2013Berkeley Shenzhen Institute","ror":"https://ror.org/02hhwwz98","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210114105","https://openalex.org/I95457486","https://openalex.org/I99065089"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zunnan Xu","raw_affiliation_strings":["Tsinghua University,Tsinghua Shenzhen International Graduate School","Tsinghua Shenzhen International Graduate School, Tsinghua University"],"affiliations":[{"raw_affiliation_string":"Tsinghua University,Tsinghua Shenzhen International Graduate School","institution_ids":["https://openalex.org/I4210114105","https://openalex.org/I99065089"]},{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School, Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102788950","display_name":"Haiwei Xue","orcid":"https://orcid.org/0000-0001-7318-9682"},"institutions":[{"id":"https://openalex.org/I4210114105","display_name":"Tsinghua\u2013Berkeley Shenzhen Institute","ror":"https://ror.org/02hhwwz98","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210114105","https://openalex.org/I95457486","https://openalex.org/I99065089"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haiwei Xue","raw_affiliation_strings":["Tsinghua University,Tsinghua Shenzhen International Graduate School","Tsinghua Shenzhen International Graduate School, Tsinghua University"],"affiliations":[{"raw_affiliation_string":"Tsinghua University,Tsinghua Shenzhen International Graduate School","institution_ids":["https://openalex.org/I4210114105","https://openalex.org/I99065089"]},{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School, Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082921355","display_name":"Yongkang Cheng","orcid":"https://orcid.org/0000-0002-8849-2032"},"institutions":[{"id":"https://openalex.org/I36819085","display_name":"Northwest University","ror":"https://ror.org/00y7snj24","country_code":"US","type":"education","lineage":["https://openalex.org/I36819085"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yongkang Cheng","raw_affiliation_strings":["Northwest A&F University"],"affiliations":[{"raw_affiliation_string":"Northwest A&F University","institution_ids":["https://openalex.org/I36819085"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017818991","display_name":"Shaoli Huang","orcid":"https://orcid.org/0000-0002-1445-3196"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shaoli Huang","raw_affiliation_strings":["Tencent AI Lab"],"affiliations":[{"raw_affiliation_string":"Tencent AI Lab","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102023771","display_name":"Mingming Gong","orcid":"https://orcid.org/0000-0001-7147-5589"},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]},{"id":"https://openalex.org/I165779595","display_name":"University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AE","AU"],"is_corresponding":false,"raw_author_name":"Mingming Gong","raw_affiliation_strings":["University of Melbourne","Mohamed Bin Zayed University of Artificial Intelligence"],"affiliations":[{"raw_affiliation_string":"University of Melbourne","institution_ids":["https://openalex.org/I165779595"]},{"raw_affiliation_string":"Mohamed Bin Zayed University of Artificial Intelligence","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102869280","display_name":"Zhiyong Wu","orcid":"https://orcid.org/0000-0001-8533-0524"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]},{"id":"https://openalex.org/I4210114105","display_name":"Tsinghua\u2013Berkeley Shenzhen Institute","ror":"https://ror.org/02hhwwz98","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210114105","https://openalex.org/I95457486","https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiyong Wu","raw_affiliation_strings":["Tsinghua University,Tsinghua Shenzhen International Graduate School","Tsinghua Shenzhen International Graduate School, Tsinghua University"],"affiliations":[{"raw_affiliation_string":"Tsinghua University,Tsinghua Shenzhen International Graduate School","institution_ids":["https://openalex.org/I4210114105","https://openalex.org/I99065089"]},{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School, Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5113092484"],"corresponding_institution_ids":["https://openalex.org/I4210114105","https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":5.4055,"has_fulltext":false,"cited_by_count":16,"citation_normalized_percentile":{"value":0.96195562,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"7945","last_page":"7949"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12290","display_name":"Human Motion and Animation","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12290","display_name":"Human Motion and Animation","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8235198855400085},{"id":"https://openalex.org/keywords/gesture","display_name":"Gesture","score":0.7893720865249634},{"id":"https://openalex.org/keywords/naturalness","display_name":"Naturalness","score":0.660996675491333},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.6449882388114929},{"id":"https://openalex.org/keywords/utterance","display_name":"Utterance","score":0.5243297219276428},{"id":"https://openalex.org/keywords/motion","display_name":"Motion (physics)","score":0.4548051059246063},{"id":"https://openalex.org/keywords/speech-synthesis","display_name":"Speech synthesis","score":0.4522474706172943},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.43982747197151184},{"id":"https://openalex.org/keywords/generative-model","display_name":"Generative model","score":0.426134318113327},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.32887643575668335}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8235198855400085},{"id":"https://openalex.org/C207347870","wikidata":"https://www.wikidata.org/wiki/Q371174","display_name":"Gesture","level":2,"score":0.7893720865249634},{"id":"https://openalex.org/C134537474","wikidata":"https://www.wikidata.org/wiki/Q17144832","display_name":"Naturalness","level":2,"score":0.660996675491333},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6449882388114929},{"id":"https://openalex.org/C2775852435","wikidata":"https://www.wikidata.org/wiki/Q258403","display_name":"Utterance","level":2,"score":0.5243297219276428},{"id":"https://openalex.org/C104114177","wikidata":"https://www.wikidata.org/wiki/Q79782","display_name":"Motion (physics)","level":2,"score":0.4548051059246063},{"id":"https://openalex.org/C14999030","wikidata":"https://www.wikidata.org/wiki/Q16346","display_name":"Speech synthesis","level":2,"score":0.4522474706172943},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43982747197151184},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.426134318113327},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.32887643575668335},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp48485.2024.10447978","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10447978","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1121271761","display_name":null,"funder_award_id":"Program","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1888327543","display_name":null,"funder_award_id":"15001","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2087396116","display_name":null,"funder_award_id":"China","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2123497280","display_name":null,"funder_award_id":"202106","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2603945996","display_name":null,"funder_award_id":"62076144","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2981938667","display_name":null,"funder_award_id":"Shenzhen","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3317480652","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3757194791","display_name":null,"funder_award_id":"JCYJ20","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5898128861","display_name":null,"funder_award_id":"61405150","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5939423041","display_name":null,"funder_award_id":"Technology","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5994120800","display_name":null,"funder_award_id":"Natural","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320316083","display_name":"Tencent","ror":"https://ror.org/00hhjss72"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":39,"referenced_works":["https://openalex.org/W2064076387","https://openalex.org/W2922298118","https://openalex.org/W2978956737","https://openalex.org/W3025281718","https://openalex.org/W3036167779","https://openalex.org/W3083173864","https://openalex.org/W3099073275","https://openalex.org/W3122282950","https://openalex.org/W3125775899","https://openalex.org/W3131922516","https://openalex.org/W3209984917","https://openalex.org/W4221142137","https://openalex.org/W4249736682","https://openalex.org/W4288099666","https://openalex.org/W4310883287","https://openalex.org/W4312635677","https://openalex.org/W4321351659","https://openalex.org/W4323076533","https://openalex.org/W4364377334","https://openalex.org/W4372346581","https://openalex.org/W4377010269","https://openalex.org/W4385245566","https://openalex.org/W4385284180","https://openalex.org/W4385764101","https://openalex.org/W4386065807","https://openalex.org/W4386075888","https://openalex.org/W4386270206","https://openalex.org/W4386755513","https://openalex.org/W4387421378","https://openalex.org/W4388505252","https://openalex.org/W4390873580","https://openalex.org/W4393148235","https://openalex.org/W6771626834","https://openalex.org/W6779823529","https://openalex.org/W6791353385","https://openalex.org/W6840815571","https://openalex.org/W6844223692","https://openalex.org/W6846857096","https://openalex.org/W6850082022"],"related_works":["https://openalex.org/W4391272374","https://openalex.org/W1914543332","https://openalex.org/W2946856121","https://openalex.org/W40885451","https://openalex.org/W2108985546","https://openalex.org/W4365211920","https://openalex.org/W2081919107","https://openalex.org/W2433276473","https://openalex.org/W3014948380","https://openalex.org/W2535215250"],"abstract_inverted_index":{"Current":[0],"talking":[1],"avatars":[2],"mostly":[3],"generate":[4],"co-speech":[5,27,76],"gestures":[6,104],"based":[7,34],"on":[8,26,35],"audio":[9],"and":[10,47,78,105,147,160,167],"text":[11],"of":[12,20,62,72,101],"the":[13,17,21,60,66,70,83,125,128],"utterance,":[14],"without":[15],"considering":[16],"non-speaking":[18],"motion":[19,95,114,150],"speaker.":[22],"Furthermore,":[23],"previous":[24],"works":[25],"gesture":[28,37],"generation":[29,71,96],"have":[30],"designed":[31],"network":[32],"structures":[33],"individual":[36],"datasets,":[38],"which":[39],"results":[40],"in":[41,127],"limited":[42],"data":[43,110],"volume,":[44],"compromised":[45],"generalizability,":[46],"restricted":[48],"speaker":[49,85,94,162],"movements.":[50,163],"To":[51],"tackle":[52],"these":[53],"issues,":[54],"we":[55,88,118,137],"introduce":[56],"FreeTalker,":[57],"which,":[58],"to":[59,122,131],"best":[61],"our":[63,156],"knowledge,":[64],"is":[65],"first":[67],"framework":[68],"for":[69,93],"both":[73,102],"spontaneous":[74],"(e.g.,":[75,80],"gesture)":[77],"non-spontaneous":[79],"moving":[81],"around":[82],"podium)":[84],"motions.":[86],"Specifically,":[87],"train":[89],"a":[90,140,144],"diffusion-based":[91],"model":[92],"that":[97,142,155],"employs":[98],"unified":[99],"representations":[100],"speech-driven":[103],"text-driven":[106],"motions,":[107],"utilizing":[108],"heterogeneous":[109],"sourced":[111],"from":[112],"various":[113],"datasets.":[115],"During":[116],"inference,":[117],"utilize":[119,138],"classifier-free":[120],"guidance":[121],"highly":[123],"control":[124],"style":[126],"clips.":[129],"Additionally,":[130],"create":[132],"smooth":[133],"transitions":[134],"between":[135],"clips,":[136],"DoubleTake,":[139],"method":[141,157],"leverages":[143],"generative":[145],"prior":[146],"ensures":[148],"seamless":[149],"blending.":[151],"Extensive":[152],"experiments":[153],"show":[154],"generates":[158],"natural":[159],"controllable":[161],"Our":[164],"code,":[165],"model,":[166],"demo":[168],"are":[169,170],"available":[171],"at":[172],"https://youngseng.github.io/FreeTalker/.":[173]},"counts_by_year":[{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":7}],"updated_date":"2026-03-18T14:38:29.013473","created_date":"2025-10-10T00:00:00"}