{"id":"https://openalex.org/W4403780722","doi":"https://doi.org/10.1145/3664647.3681697","title":"Generative Expressive Conversational Speech Synthesis","display_name":"Generative Expressive Conversational Speech Synthesis","alternate_titles":[],"publication_year":2024,"publication_date":"2024-10-26","ids":{"openalex":"https://openalex.org/W4403780722","doi":"https://doi.org/10.1145/3664647.3681697"},"language":"en","primary_location":{"id":"doi:10.1145/3664647.3681697","doi":"https://doi.org/10.1145/3664647.3681697","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3681697","pdf_url":null,"source":{"id":"https://openalex.org/S4363608757","display_name":"ACM International Conference on Multimedia (ACM MM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W4403780722"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2407.21491","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5082746577","display_name":"Rui Liu","orcid":"https://orcid.org/0000-0003-4524-7413","observed_orcids":["https://orcid.org/0000-0003-4524-7413"]},"institutions":[{"id":"https://openalex.org/I2722730","display_name":"Inner Mongolia University","ror":"https://ror.org/0106qb496","country_code":"CN","type":"education","lineage":["https://openalex.org/I2722730"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rui Liu","raw_affiliation_strings":["Inner Mongolia University, Hohhot, China"],"raw_orcid":"https://orcid.org/0000-0003-4524-7413","affiliations":[{"raw_affiliation_string":"Inner Mongolia University, Hohhot, China","institution_ids":["https://openalex.org/I2722730"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100708728","display_name":"Yifan Hu","orcid":"https://orcid.org/0009-0008-2276-1456","observed_orcids":["https://orcid.org/0009-0008-2276-1456"]},"institutions":[{"id":"https://openalex.org/I2722730","display_name":"Inner Mongolia University","ror":"https://ror.org/0106qb496","country_code":"CN","type":"education","lineage":["https://openalex.org/I2722730"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yifan Hu","raw_affiliation_strings":["Inner Mongolia University, Hohhot, China"],"raw_orcid":"https://orcid.org/0009-0008-2276-1456","affiliations":[{"raw_affiliation_string":"Inner Mongolia University, Hohhot, China","institution_ids":["https://openalex.org/I2722730"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108582272","display_name":"Yi Ren","orcid":"https://orcid.org/0000-0002-9160-3848","observed_orcids":["https://orcid.org/0000-0002-9160-3848"]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yi Ren","raw_affiliation_strings":["ByteDance, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0002-9160-3848","affiliations":[{"raw_affiliation_string":"ByteDance, Singapore, Singapore","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100446849","display_name":"Xiang Yin","orcid":"https://orcid.org/0000-0003-0472-2783","observed_orcids":["https://orcid.org/0000-0003-0472-2783","https://orcid.org/0000-0003-0741-4831"]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiang Yin","raw_affiliation_strings":["ByteDance, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-0472-2783","affiliations":[{"raw_affiliation_string":"ByteDance, Shanghai, China","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032690182","display_name":"Haizhou Li","orcid":"https://orcid.org/0000-0001-9158-9401","observed_orcids":["https://orcid.org/0000-0001-9158-9401"]},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]},{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]},{"id":"https://openalex.org/I4210099586","display_name":"Shenzhen Research Institute of Big Data","ror":"https://ror.org/00z1gwf89","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210099586"]},{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN","HK","SG"],"is_corresponding":false,"raw_author_name":"Haizhou Li","raw_affiliation_strings":["SRIBD, School of Data Science, The Chinese University of Hong Kong & National University of Singapore, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0001-9158-9401","affiliations":[{"raw_affiliation_string":"SRIBD, School of Data Science, The Chinese University of Hong Kong & National University of Singapore, Shenzhen, China","institution_ids":["https://openalex.org/I165932596","https://openalex.org/I177725633","https://openalex.org/I4210099586","https://openalex.org/I4210116924"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.9515,"has_fulltext":true,"cited_by_count":51,"citation_normalized_percentile":{"value":0.95202005,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"4187","last_page":"4196"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.00279999990016222,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.0005000000237487257,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/conversational-speech-synthesis","display_name":"conversational speech synthesis","score":0.996999979019165},{"id":"https://openalex.org/keywords/expressive-speech-synthesis","display_name":"expressive speech synthesis","score":0.8629999756813049},{"id":"https://openalex.org/keywords/dialogue-context-modeling","display_name":"dialogue context modeling","score":0.6669999957084656},{"id":"https://openalex.org/keywords/vits","display_name":"VITS","score":0.5049999952316284},{"id":"https://openalex.org/keywords/speech-style","display_name":"speech style","score":0.35100001096725464}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7877883911132812},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.6673979163169861},{"id":"https://openalex.org/C14999030","wikidata":"https://www.wikidata.org/wiki/Q16346","display_name":"Speech synthesis","level":2,"score":0.5960658192634583},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5870063900947571},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.513836681842804},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42386436462402344}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3664647.3681697","doi":"https://doi.org/10.1145/3664647.3681697","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3681697","pdf_url":null,"source":{"id":"https://openalex.org/S4363608757","display_name":"ACM International Conference on Multimedia (ACM MM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2407.21491","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.21491","pdf_url":"https://arxiv.org/pdf/2407.21491","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2407.21491","doi":"https://doi.org/10.48550/arxiv.2407.21491","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2407.21491","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2407.21491","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.21491","pdf_url":"https://arxiv.org/pdf/2407.21491","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G101729752","display_name":null,"funder_award_id":"21300-231510","funder_id":"https://openalex.org/F4320322216","funder_display_name":"Inner Mongolia University"},{"id":"https://openalex.org/G1417164178","display_name":null,"funder_award_id":"11200-5223737","funder_id":"https://openalex.org/F4320322216","funder_display_name":"Inner Mongolia University"},{"id":"https://openalex.org/G2799040894","display_name":null,"funder_award_id":"62271432","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4617505510","display_name":null,"funder_award_id":"62206136","funder_id":"https://openalex.org/F4320331088","funder_display_name":"Natural Science Foundation for Young Scientists of Shanxi Province"},{"id":"https://openalex.org/G600008135","display_name":null,"funder_award_id":"62206136","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322216","display_name":"Inner Mongolia University","ror":"https://ror.org/0106qb496"},{"id":"https://openalex.org/F4320331088","display_name":"Natural Science Foundation for Young Scientists of Shanxi Province","ror":null},{"id":"https://openalex.org/F4320335581","display_name":"Young Scientists Fund","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403780722.pdf","grobid_xml":"https://content.openalex.org/works/W4403780722.grobid-xml"},"referenced_works_count":48,"referenced_works":["https://openalex.org/W2146334809","https://openalex.org/W2895615590","https://openalex.org/W2898035588","https://openalex.org/W2948972794","https://openalex.org/W2972359262","https://openalex.org/W2988937804","https://openalex.org/W3033411150","https://openalex.org/W3094002217","https://openalex.org/W3095319910","https://openalex.org/W3115598554","https://openalex.org/W3129009457","https://openalex.org/W3151309757","https://openalex.org/W3155340931","https://openalex.org/W3169320628","https://openalex.org/W3169874582","https://openalex.org/W3195592874","https://openalex.org/W4221163454","https://openalex.org/W4225300652","https://openalex.org/W4281397735","https://openalex.org/W4283067649","https://openalex.org/W4286273498","https://openalex.org/W4304013787","https://openalex.org/W4307680525","https://openalex.org/W4308852802","https://openalex.org/W4313679638","https://openalex.org/W4318903120","https://openalex.org/W4372260474","https://openalex.org/W4375868859","https://openalex.org/W4382202703","https://openalex.org/W4384615685","https://openalex.org/W4385823169","https://openalex.org/W4387595589","https://openalex.org/W4387968649","https://openalex.org/W4387969386","https://openalex.org/W4389524500","https://openalex.org/W4390690192","https://openalex.org/W4391021666","https://openalex.org/W4392384650","https://openalex.org/W4392902952","https://openalex.org/W4392908891","https://openalex.org/W4392931281","https://openalex.org/W4393160807","https://openalex.org/W4402683976","https://openalex.org/W4403334432","https://openalex.org/W4405709862","https://openalex.org/W4405710078","https://openalex.org/W4415800775","https://openalex.org/W7234005527"],"related_works":["https://openalex.org/W4406560414","https://openalex.org/W4408345868","https://openalex.org/W4412888749","https://openalex.org/W7169797390","https://openalex.org/W4405709767","https://openalex.org/W4415536841","https://openalex.org/W4408345994","https://openalex.org/W7171538085","https://openalex.org/W3151309757","https://openalex.org/W4405710078"],"abstract_inverted_index":{"Conversational":[0],"Speech":[1],"Synthesis":[2],"(CSS)":[3],"aims":[4],"to":[5,29,39,54,67,105,146,149],"express":[6],"a":[7,16,80,107,154,185],"target":[8],"utterance":[9],"with":[10,184],"the":[11,47,55,75,89,93,113,119,132,136,143,150,196,199,202],"proper":[12],"speaking":[13],"style":[14,127],"in":[15,168,222],"user-agent":[17,109],"conversation":[18],"setting.":[19],"Existing":[20],"CSS":[21,84,157,219],"methods":[22],"employ":[23],"effective":[24],"multi-modal":[25],"context":[26],"modeling":[27],"techniques":[28],"achieve":[30],"empathy":[31],"understanding":[32],"and":[33,44,101,126,171,181,201,209,226,231],"expression.":[34],"However,":[35],"they":[36,64],"often":[37,65],"need":[38],"design":[40],"complex":[41],"network":[42],"architectures":[43],"meticulously":[45],"optimize":[46],"modules":[48],"within":[49],"them.":[50],"In":[51],"addition,":[52],"due":[53],"limitations":[56],"of":[57,92,115,129,188,198,204,224],"small-scale":[58],"datasets":[59],"containing":[60],"scripted":[61],"recording":[62],"styles,":[63],"fail":[66],"simulate":[68],"real":[69],"natural":[70],"conversational":[71,138,166],"styles.":[72],"To":[73],"address":[74],"above":[76],"issues,":[77],"we":[78,117,152],"propose":[79,153],"novel":[81],"generative":[82],"expressive":[83,137],"system,":[85],"termed":[86],"GPT-Talker.We":[87],"transform":[88],"multimodal":[90],"information":[91],"multi-turn":[94],"dialogue":[95,110],"history":[96],"into":[97],"discrete":[98],"token":[99,120],"sequences":[100],"seamlessly":[102],"integrate":[103],"them":[104],"form":[106],"comprehensive":[108,193],"context.":[111],"Leveraging":[112],"power":[114],"GPT,":[116],"predict":[118],"sequence,":[121],"that":[122,161,213],"includes":[123,162],"both":[124,163,179],"semantic":[125],"knowledge,":[128],"response":[130],"for":[131],"agent.":[133],"After":[134],"that,":[135],"speech":[139,167],"is":[140],"synthesized":[141],"by":[142],"conversation-enriched":[144],"VITS":[145],"deliver":[147],"feedback":[148],"user.Furthermore,":[151],"large-scale":[155],"Natural":[156],"Dataset":[158],"called":[159],"NCSSD,":[160],"naturally":[164],"recorded":[165],"improvised":[169],"styles":[170],"dialogues":[172],"extracted":[173],"from":[174],"TV":[175],"shows.":[176],"It":[177],"encompasses":[178],"Chinese":[180],"English":[182],"languages,":[183],"total":[186],"duration":[187],"236":[189],"hours.":[190],"We":[191],"conducted":[192],"experiments":[194],"on":[195],"reliability":[197],"NCSSD":[200],"effectiveness":[203],"our":[205,214],"GPT-Talker.":[206],"Both":[207],"subjective":[208],"objective":[210],"evaluations":[211],"demonstrate":[212],"model":[215],"outperforms":[216],"other":[217],"state-of-the-art":[218],"systems":[220],"significantly":[221],"terms":[223],"naturalness":[225],"expressiveness.":[227],"The":[228],"Code,":[229],"Dataset,":[230],"Pre-trained":[232],"Model":[233],"are":[234],"available":[235],"at:":[236],"https://github.com/AI-S2-Lab/GPT-Talker.":[237]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4403780722","counts_by_year":[{"year":2026,"cited_by_count":26},{"year":2025,"cited_by_count":23},{"year":2024,"cited_by_count":2}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2024-10-26T00:00:00"}
