{"id":"https://openalex.org/W4403780871","doi":"https://doi.org/10.1145/3664647.3681465","title":"UniStyle: Unified Style Modeling for Speaking Style Captioning and Stylistic Speech Synthesis","display_name":"UniStyle: Unified Style Modeling for Speaking Style Captioning and Stylistic Speech Synthesis","publication_year":2024,"publication_date":"2024-10-26","ids":{"openalex":"https://openalex.org/W4403780871","doi":"https://doi.org/10.1145/3664647.3681465"},"language":"en","primary_location":{"id":"doi:10.1145/3664647.3681465","doi":"https://doi.org/10.1145/3664647.3681465","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3681465","pdf_url":null,"source":{"id":"https://openalex.org/S4363608757","display_name":"ACM International Conference on Multimedia (ACM MM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101883211","display_name":"Xinfa Zhu","orcid":"https://orcid.org/0000-0001-9275-523X","observed_orcids":["https://orcid.org/0000-0001-9275-523X"]},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xinfa Zhu","raw_affiliation_strings":["Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0001-9275-523X","affiliations":[{"raw_affiliation_string":"Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Wenjie Tian","orcid":"https://orcid.org/0009-0009-3859-0927","observed_orcids":["https://orcid.org/0009-0009-3859-0927"]},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenjie Tian","raw_affiliation_strings":["Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0009-0009-3859-0927","affiliations":[{"raw_affiliation_string":"Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021472684","display_name":"Xinsheng Wang","orcid":"https://orcid.org/0000-0003-1826-7419","observed_orcids":["https://orcid.org/0000-0003-1826-7419"]},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]},{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xinsheng Wang","raw_affiliation_strings":["Microsoft, Beijing, China","Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0003-1826-7419","affiliations":[{"raw_affiliation_string":"Microsoft, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]},{"raw_affiliation_string":"Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101820632","display_name":"Lei He","orcid":"https://orcid.org/0009-0009-9870-8739","observed_orcids":["https://orcid.org/0009-0009-9870-8739"]},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei He","raw_affiliation_strings":["Microsoft, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0009-9870-8739","affiliations":[{"raw_affiliation_string":"Microsoft, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yujia Xiao","orcid":"https://orcid.org/0009-0009-5624-1704","observed_orcids":["https://orcid.org/0009-0009-5624-1704"]},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yujia Xiao","raw_affiliation_strings":["Microsoft, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0009-5624-1704","affiliations":[{"raw_affiliation_string":"Microsoft, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100442237","display_name":"Xi Wang","orcid":"https://orcid.org/0000-0002-0434-7939","observed_orcids":["https://orcid.org/0000-0002-0434-7939"]},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]},{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xi Wang","raw_affiliation_strings":["Microsoft, Beijing, China","Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0002-0434-7939","affiliations":[{"raw_affiliation_string":"Microsoft, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]},{"raw_affiliation_string":"Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101522530","display_name":"Xu Tan","orcid":"https://orcid.org/0000-0001-5631-0639","observed_orcids":["https://orcid.org/0000-0001-5631-0639"]},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xu Tan","raw_affiliation_strings":["Microsoft, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-5631-0639","affiliations":[{"raw_affiliation_string":"Microsoft, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100329353","display_name":"Sheng Zhao","orcid":"https://orcid.org/0000-0002-9624-5381","observed_orcids":["https://orcid.org/0000-0002-9624-5381","https://orcid.org/0000-0003-4030-7368"]},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sheng Zhao","raw_affiliation_strings":["Microsoft, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9624-5381","affiliations":[{"raw_affiliation_string":"Microsoft, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100668966","display_name":"Lei Xie","orcid":"https://orcid.org/0000-0001-8234-0823","observed_orcids":["https://orcid.org/0000-0001-8234-0823"]},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei Xie","raw_affiliation_strings":["Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0001-8234-0823","affiliations":[{"raw_affiliation_string":"Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.8662,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.74713605,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"7513","last_page":"7522"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.0008999999845400453,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/speech-emotion","display_name":"speech emotion","score":0.335999995470047},{"id":"https://openalex.org/keywords/semi-supervised-learning","display_name":"semi-supervised learning","score":0.9169999957084656},{"id":"https://openalex.org/keywords/two-stage-training","display_name":"two-stage training","score":0.8299999833106995},{"id":"https://openalex.org/keywords/zero-shot-speech-synthesis","display_name":"zero-shot speech synthesis","score":0.7039999961853027},{"id":"https://openalex.org/keywords/expressive-speech","display_name":"expressive speech","score":0.6600000262260437}],"concepts":[{"id":"https://openalex.org/C2776445246","wikidata":"https://www.wikidata.org/wiki/Q1792644","display_name":"Style (visual arts)","level":2,"score":0.8632208704948425},{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.7702061533927917},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7085228562355042},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5316327810287476},{"id":"https://openalex.org/C14999030","wikidata":"https://www.wikidata.org/wiki/Q16346","display_name":"Speech synthesis","level":2,"score":0.4572437107563019},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.42087340354919434},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.40759336948394775},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3167286813259125},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.08319643139839172},{"id":"https://openalex.org/C124952713","wikidata":"https://www.wikidata.org/wiki/Q8242","display_name":"Literature","level":1,"score":0.04572129249572754},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.03711223602294922},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3664647.3681465","doi":"https://doi.org/10.1145/3664647.3681465","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3681465","pdf_url":null,"source":{"id":"https://openalex.org/S4363608757","display_name":"ACM International Conference on Multimedia (ACM MM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W398859631","https://openalex.org/W1956340063","https://openalex.org/W2560674852","https://openalex.org/W2787378487","https://openalex.org/W2995181338","https://openalex.org/W3008039831","https://openalex.org/W3160329778","https://openalex.org/W3169320628","https://openalex.org/W3198048667","https://openalex.org/W3209984917","https://openalex.org/W4210777104","https://openalex.org/W4226421465","https://openalex.org/W4242982359","https://openalex.org/W4285106979","https://openalex.org/W4372260509","https://openalex.org/W4379033883","https://openalex.org/W4385822787","https://openalex.org/W4391620705","https://openalex.org/W4393147046","https://openalex.org/W4402669711"],"related_works":["https://openalex.org/W4392903700","https://openalex.org/W4417004072","https://openalex.org/W7168371421","https://openalex.org/W4406461727","https://openalex.org/W4406861371","https://openalex.org/W4385822787","https://openalex.org/W7134969975","https://openalex.org/W7138419101","https://openalex.org/W4402982072","https://openalex.org/W4319586674"],"abstract_inverted_index":{"Understanding":[0],"the":[1,6,9,56,81,86,99,109,121,127],"speaking":[2,35,59,160,172],"style,":[3],"such":[4],"as":[5,49,106],"emotion":[7],"of":[8,58,69,80,101,111,129],"interlocutor's":[10],"speech,":[11],"and":[12,34,62,72,94,108,136,163,171],"responding":[13],"with":[14,104,120,167],"speech":[15,32,64,76,92,105,118,122,166],"in":[16,24,159,174],"an":[17,44],"appropriate":[18],"style":[19,36,60,74,112,161],"is":[20,51,83],"a":[21,70,73,134,175],"natural":[22],"occurrence":[23],"human":[25],"conversations.":[26],"However,":[27],"technically,":[28],"existing":[29],"research":[30],"on":[31,150],"synthesis":[33,119],"captioning":[37,61,162],"typically":[38],"proceeds":[39],"independently.":[40],"In":[41],"this":[42],"work,":[43],"innovative":[45],"framework,":[46],"referred":[47],"to":[48,53,84,125],"UniStyle,":[50],"proposed":[52],"incorporate":[54],"both":[55],"capabilities":[57],"style-controllable":[63],"synthesizing.":[65],"Specifically,":[66],"UniStyle":[67,155],"consists":[68],"UniConnector":[71,82],"prompt-based":[75],"generator.":[77,123],"The":[78],"role":[79],"bridge":[85],"gap":[87],"between":[88],"different":[89],"modalities,":[90],"namely":[91],"audio":[93],"text":[95,102,115],"descriptions.":[96],"It":[97],"enables":[98],"generation":[100],"descriptions":[103,116],"input":[107],"creation":[110],"representations":[113],"from":[114],"for":[117],"Besides,":[124],"overcome":[126],"issue":[128],"data":[130,142],"scarcity,":[131],"we":[132],"propose":[133],"two-stage":[135],"semi-supervised":[137],"training":[138],"strategy,":[139],"which":[140],"reduces":[141],"requirements":[143],"while":[144],"boosting":[145],"performance.":[146],"Extensive":[147],"experiments":[148],"conducted":[149],"open-source":[151],"corpora":[152],"demonstrate":[153],"that":[154],"achieves":[156],"state-of-the-art":[157],"performance":[158],"synthesizes":[164],"expressive":[165],"various":[168],"speaker":[169],"timbres":[170],"styles":[173],"zero-shot":[176],"manner.":[177]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2}],"updated_date":"2026-10-14T16:39:01.820549","created_date":"2024-10-26T00:00:00"}