{"id":"https://openalex.org/W7156000539","doi":"https://doi.org/10.48550/arxiv.2604.22209","title":"UniSonate: A Unified Model for Speech, Music, and Sound Effect Generation with Text Instructions","display_name":"UniSonate: A Unified Model for Speech, Music, and Sound Effect Generation with Text Instructions","publication_year":2026,"publication_date":"2026-04-24","ids":{"openalex":"https://openalex.org/W7156000539","doi":"https://doi.org/10.48550/arxiv.2604.22209"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.22209","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.22209","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.22209","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134741057","display_name":"Chunyu Qiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Qiang, Chunyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134704865","display_name":"Xiaopeng Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xiaopeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134728857","display_name":"Kang Yin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Kang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134704673","display_name":"Yuzhe Liang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liang, Yuzhe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134666404","display_name":"Yuxin Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Yuxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134698103","display_name":"Teng Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Teng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134693380","display_name":"Ziyu Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Ziyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134674014","display_name":"Tianrui Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Tianrui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134706622","display_name":"Cheng Gong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gong, Cheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079946802","display_name":"Yushen Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yushen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134713226","display_name":"Ruibo Fu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Ruibo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134665758","display_name":"Chen Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134668293","display_name":"Longbiao Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Longbiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5134659538","display_name":"Jianwu Dang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dang, Jianwu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":14,"corresponding_author_ids":["https://openalex.org/A5134741057"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.19020000100135803,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.19020000100135803,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.15919999778270721,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.12690000236034393,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5171999931335449},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.4880000054836273},{"id":"https://openalex.org/keywords/coherence","display_name":"Coherence (philosophical gambling strategy)","score":0.47130000591278076},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"Natural language understanding","score":0.39259999990463257},{"id":"https://openalex.org/keywords/generative-model","display_name":"Generative model","score":0.3808000087738037},{"id":"https://openalex.org/keywords/fidelity","display_name":"Fidelity","score":0.376800000667572},{"id":"https://openalex.org/keywords/unified-model","display_name":"Unified Model","score":0.37459999322891235},{"id":"https://openalex.org/keywords/high-fidelity","display_name":"High fidelity","score":0.3702000081539154},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.3368000090122223}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7748000025749207},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5171999931335449},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.4880000054836273},{"id":"https://openalex.org/C2781181686","wikidata":"https://www.wikidata.org/wiki/Q4226068","display_name":"Coherence (philosophical gambling strategy)","level":2,"score":0.47130000591278076},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4505999982357025},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.39259999990463257},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.3808000087738037},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.376800000667572},{"id":"https://openalex.org/C45493050","wikidata":"https://www.wikidata.org/wiki/Q7884934","display_name":"Unified Model","level":2,"score":0.37459999322891235},{"id":"https://openalex.org/C113364801","wikidata":"https://www.wikidata.org/wiki/Q26674","display_name":"High fidelity","level":2,"score":0.3702000081539154},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3578999936580658},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.3368000090122223},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3294999897480011},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.2989000082015991},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.29179999232292175},{"id":"https://openalex.org/C9940772","wikidata":"https://www.wikidata.org/wiki/Q557399","display_name":"Psychoacoustics","level":3,"score":0.28839999437332153},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.2840999960899353},{"id":"https://openalex.org/C2780646970","wikidata":"https://www.wikidata.org/wiki/Q6980787","display_name":"Natural sounds","level":2,"score":0.27950000762939453},{"id":"https://openalex.org/C150202949","wikidata":"https://www.wikidata.org/wiki/Q107602","display_name":"Pointer (user interface)","level":2,"score":0.2791999876499176},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.2734000086784363},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.27070000767707825},{"id":"https://openalex.org/C155635449","wikidata":"https://www.wikidata.org/wiki/Q4674699","display_name":"Acoustic model","level":3,"score":0.26930001378059387},{"id":"https://openalex.org/C14999030","wikidata":"https://www.wikidata.org/wiki/Q16346","display_name":"Speech synthesis","level":2,"score":0.258899986743927},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.2583000063896179},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.257099986076355}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.22209","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.22209","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.22209","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.22209","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"score":0.8174575567245483,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Generative":[0],"audio":[1,154],"modeling":[2],"has":[3],"largely":[4],"been":[5],"fragmented":[6],"into":[7,89],"specialized":[8],"tasks,":[9],"text-to-speech":[10],"(TTS),":[11],"text-to-music":[12],"(TTM),":[13],"and":[14,40,61,133,160],"text-to-audio":[15],"(TTA),":[16],"each":[17],"operating":[18],"under":[19],"heterogeneous":[20],"control":[21,98],"paradigms.":[22],"Unifying":[23],"these":[24],"modalities":[25],"remains":[26],"a":[27,52,65,78,90,100,108],"fundamental":[28],"challenge":[29],"due":[30],"to":[31,164],"the":[32],"intrinsic":[33],"dissonance":[34],"between":[35],"structured":[36,91],"semantic":[37],"representations":[38],"(speech/music)":[39],"unstructured":[41,86],"acoustic":[42],"textures":[43],"(sound":[44],"effects).":[45],"In":[46],"this":[47,113],"paper,":[48],"we":[49,76,145],"introduce":[50],"UniSonate,":[51],"unified":[53],"flow-matching":[54],"framework":[55],"capable":[56],"of":[57],"synthesizing":[58],"speech,":[59],"music,":[60],"sound":[62],"effects":[63],"through":[64],"standardized,":[66],"reference-free":[67],"natural":[68],"language":[69],"instruction":[70],"interface.":[71],"To":[72],"reconcile":[73],"structural":[74,158],"disparities,":[75],"propose":[77],"novel":[79],"dynamic":[80],"token":[81],"injection":[82],"mechanism":[83],"that":[84,123],"projects":[85],"environmental":[87],"sounds":[88],"temporal":[92],"latent":[93],"space,":[94],"enabling":[95],"precise":[96],"duration":[97],"within":[99],"phoneme-driven":[101],"Multimodal":[102],"Diffusion":[103],"Transformer":[104],"(MM-DiT).":[105],"Coupled":[106],"with":[107],"multi-stage":[109],"curriculum":[110],"learning":[111],"strategy,":[112],"approach":[114],"effectively":[115],"mitigates":[116],"cross-modal":[117],"optimization":[118],"conflicts.":[119],"Extensive":[120],"experiments":[121],"demonstrate":[122],"UniSonate":[124],"achieves":[125],"state-of-the-art":[126],"performance":[127],"in":[128,142],"instruction-based":[129],"TTS":[130],"(WER":[131],"1.47%)":[132],"TTM":[134],"(SongEval":[135],"Coherence":[136],"3.18),":[137],"while":[138],"maintaining":[139],"competitive":[140],"fidelity":[141],"TTA.":[143],"Crucially,":[144],"observe":[146],"positive":[147],"transfer,":[148],"where":[149],"joint":[150],"training":[151],"on":[152],"diverse":[153],"data":[155],"significantly":[156],"enhances":[157],"coherence":[159],"prosodic":[161],"expressiveness":[162],"compared":[163],"single-task":[165],"baselines.":[166],"Audio":[167],"samples":[168],"are":[169],"available":[170],"at":[171],"https://qiangchunyu.github.io/UniSonate/.":[172]},"counts_by_year":[],"updated_date":"2026-04-28T06:12:00.211691","created_date":"2026-04-28T00:00:00"}