{"id":"https://openalex.org/W4386764084","doi":"https://doi.org/10.1109/waspaa58266.2023.10248160","title":"CLIPSonic: Text-to-Audio Synthesis with Unlabeled Videos and Pretrained Language-Vision Models","display_name":"CLIPSonic: Text-to-Audio Synthesis with Unlabeled Videos and Pretrained Language-Vision Models","publication_year":2023,"publication_date":"2023-09-15","ids":{"openalex":"https://openalex.org/W4386764084","doi":"https://doi.org/10.1109/waspaa58266.2023.10248160"},"language":"en","primary_location":{"id":"doi:10.1109/waspaa58266.2023.10248160","is_oa":false,"landing_page_url":"https://doi.org/10.1109/waspaa58266.2023.10248160","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Workshop on Applications of Signal Processing to Audio and Acoustics (WASPAA)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028026494","display_name":"Hao\u2010Wen Dong","orcid":"https://orcid.org/0000-0002-1020-0001"},"institutions":[{"id":"https://openalex.org/I116121188","display_name":"Dolby (Netherlands)","ror":"https://ror.org/02kb3q578","country_code":"NL","type":"company","lineage":["https://openalex.org/I116121188","https://openalex.org/I4210093996"]},{"id":"https://openalex.org/I36258959","display_name":"University of California, San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["NL","US"],"is_corresponding":true,"raw_author_name":"Hao-Wen Dong","raw_affiliation_strings":["Dolby Laboratories","University of California, San Diego"],"affiliations":[{"raw_affiliation_string":"Dolby Laboratories","institution_ids":["https://openalex.org/I116121188"]},{"raw_affiliation_string":"University of California, San Diego","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092206184","display_name":"Xiaoyu Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I116121188","display_name":"Dolby (Netherlands)","ror":"https://ror.org/02kb3q578","country_code":"NL","type":"company","lineage":["https://openalex.org/I116121188","https://openalex.org/I4210093996"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Xiaoyu Liu","raw_affiliation_strings":["Dolby Laboratories"],"affiliations":[{"raw_affiliation_string":"Dolby Laboratories","institution_ids":["https://openalex.org/I116121188"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103137809","display_name":"Jordi Pons","orcid":"https://orcid.org/0000-0001-9603-0869"},"institutions":[{"id":"https://openalex.org/I116121188","display_name":"Dolby (Netherlands)","ror":"https://ror.org/02kb3q578","country_code":"NL","type":"company","lineage":["https://openalex.org/I116121188","https://openalex.org/I4210093996"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Jordi Pons","raw_affiliation_strings":["Dolby Laboratories"],"affiliations":[{"raw_affiliation_string":"Dolby Laboratories","institution_ids":["https://openalex.org/I116121188"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102772985","display_name":"Gautam Bhattacharya","orcid":"https://orcid.org/0000-0003-4787-0604"},"institutions":[{"id":"https://openalex.org/I116121188","display_name":"Dolby (Netherlands)","ror":"https://ror.org/02kb3q578","country_code":"NL","type":"company","lineage":["https://openalex.org/I116121188","https://openalex.org/I4210093996"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Gautam Bhattacharya","raw_affiliation_strings":["Dolby Laboratories"],"affiliations":[{"raw_affiliation_string":"Dolby Laboratories","institution_ids":["https://openalex.org/I116121188"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101557983","display_name":"Santiago Pascual","orcid":"https://orcid.org/0000-0002-8365-7387"},"institutions":[{"id":"https://openalex.org/I116121188","display_name":"Dolby (Netherlands)","ror":"https://ror.org/02kb3q578","country_code":"NL","type":"company","lineage":["https://openalex.org/I116121188","https://openalex.org/I4210093996"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Santiago Pascual","raw_affiliation_strings":["Dolby Laboratories"],"affiliations":[{"raw_affiliation_string":"Dolby Laboratories","institution_ids":["https://openalex.org/I116121188"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103092198","display_name":"Joan Serr\u00e0","orcid":"https://orcid.org/0000-0003-1303-6558"},"institutions":[{"id":"https://openalex.org/I116121188","display_name":"Dolby (Netherlands)","ror":"https://ror.org/02kb3q578","country_code":"NL","type":"company","lineage":["https://openalex.org/I116121188","https://openalex.org/I4210093996"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Joan Serr\u00e0","raw_affiliation_strings":["Dolby Laboratories"],"affiliations":[{"raw_affiliation_string":"Dolby Laboratories","institution_ids":["https://openalex.org/I116121188"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017455302","display_name":"Taylor Berg-Kirkpatrick","orcid":"https://orcid.org/0000-0002-1283-4075"},"institutions":[{"id":"https://openalex.org/I36258959","display_name":"University of California, San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Taylor Berg-Kirkpatrick","raw_affiliation_strings":["University of California,San Diego","University of California, San Diego"],"affiliations":[{"raw_affiliation_string":"University of California,San Diego","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"University of California, San Diego","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021827617","display_name":"Julian McAuley","orcid":"https://orcid.org/0000-0003-0955-7588"},"institutions":[{"id":"https://openalex.org/I36258959","display_name":"University of California, San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Julian McAuley","raw_affiliation_strings":["University of California,San Diego","University of California, San Diego"],"affiliations":[{"raw_affiliation_string":"University of California,San Diego","institution_ids":["https://openalex.org/I36258959"]},{"raw_affiliation_string":"University of California, San Diego","institution_ids":["https://openalex.org/I36258959"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5028026494"],"corresponding_institution_ids":["https://openalex.org/I116121188","https://openalex.org/I36258959"],"apc_list":null,"apc_paid":null,"fwci":3.2668,"has_fulltext":false,"cited_by_count":16,"citation_normalized_percentile":{"value":0.93258216,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.9890999794006348,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.840294599533081},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.5728592872619629},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.5686289072036743},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5397153496742249},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.5238097906112671},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4586079716682434},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3480132520198822},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3251052796840668}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.840294599533081},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.5728592872619629},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5686289072036743},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5397153496742249},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.5238097906112671},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4586079716682434},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3480132520198822},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3251052796840668}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/waspaa58266.2023.10248160","is_oa":false,"landing_page_url":"https://doi.org/10.1109/waspaa58266.2023.10248160","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Workshop on Applications of Signal Processing to Audio and Acoustics (WASPAA)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6000000238418579,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":32,"referenced_works":["https://openalex.org/W2526050071","https://openalex.org/W2962865004","https://openalex.org/W2964345931","https://openalex.org/W2972478942","https://openalex.org/W3015371781","https://openalex.org/W3176445421","https://openalex.org/W3197828817","https://openalex.org/W4224035735","https://openalex.org/W4284898017","https://openalex.org/W4288099666","https://openalex.org/W4312933868","https://openalex.org/W4318351475","https://openalex.org/W4319989813","https://openalex.org/W4367359628","https://openalex.org/W4372260310","https://openalex.org/W4372263438","https://openalex.org/W4372348103","https://openalex.org/W6779823529","https://openalex.org/W6783182287","https://openalex.org/W6788990321","https://openalex.org/W6791353385","https://openalex.org/W6802805937","https://openalex.org/W6809885388","https://openalex.org/W6838843145","https://openalex.org/W6841982715","https://openalex.org/W6845479124","https://openalex.org/W6846007759","https://openalex.org/W6847370424","https://openalex.org/W6849105126","https://openalex.org/W6849109464","https://openalex.org/W6849416043","https://openalex.org/W6849635556"],"related_works":["https://openalex.org/W1891287906","https://openalex.org/W2036807459","https://openalex.org/W2775347418","https://openalex.org/W1969923398","https://openalex.org/W2772917594","https://openalex.org/W2166024367","https://openalex.org/W2755342338","https://openalex.org/W3116076068","https://openalex.org/W2229312674","https://openalex.org/W2359001871"],"abstract_inverted_index":{"Recent":[0],"work":[1],"has":[2],"studied":[3],"text-to-audio":[4,30,163,199],"synthesis":[5,31,184,200],"using":[6,32],"large":[7],"amounts":[8],"of":[9,66,143,197,212],"paired":[10],"text-audio":[11,45],"data.":[12],"However,":[13,103],"audio":[14,64,171],"recordings":[15],"with":[16,98,110],"high-quality":[17],"text":[18,101,136],"annotations":[19],"can":[20,153,168],"be":[21],"difficult":[22],"to":[23,41,61,112,127],"acquire.":[24],"In":[25],"this":[26,117],"work,":[27],"we":[28,85,104,119,160],"approach":[29],"unlabeled":[33],"videos":[34,208],"and":[35,93,147,175,209],"pre-trained":[36],"language-vision":[37,214],"models.":[38,215],"We":[39,55],"propose":[40],"learn":[42],"the":[43,49,63,95,141,144,149,155,165,203,210],"desired":[44],"correspondence":[46,206],"by":[47,74],"leveraging":[48],"visual":[50],"modality":[51,91,156],"as":[52],"a":[53,57,67,70,75,89,99,106,122,129,134,181,187,194],"bridge.":[54],"train":[56],"conditional":[58],"diffusion":[59,96,124,151],"model":[60,97,126,167,185],"generate":[62,128,170],"track":[65],"video,":[68],"given":[69,133],"video":[71],"frame":[72],"encoded":[73],"pretrained":[76,123,150,213],"contrastive":[77],"language-image":[78],"pretraining":[79],"(CLIP)":[80],"model.":[81],"At":[82],"test":[83],"time,":[84],"first":[86],"explore":[87],"performing":[88],"zero-shot":[90],"transfer":[92,157],"condition":[94],"CLIP-encoded":[100],"query.":[102],"observe":[105],"noticeable":[107],"performance":[108,179],"drop":[109],"respect":[111],"image":[113,131,173],"queries.":[114],"To":[115],"close":[116],"gap,":[118],"further":[120],"adopt":[121],"prior":[125,152],"CLIP":[130,135],"embedding":[132],"embedding.":[137],"Our":[138],"results":[139],"show":[140],"effectiveness":[142],"proposed":[145,166],"method,":[146],"that":[148,201],"reduce":[154],"gap.":[158],"While":[159],"focus":[161],"on":[162],"synthesis,":[164],"also":[169],"from":[172],"queries,":[174],"it":[176],"shows":[177],"competitive":[178],"against":[180],"state-of-the-art":[182],"image-to-audio":[183],"in":[186,207],"subjective":[188],"listening":[189],"test.":[190],"This":[191],"study":[192],"offers":[193],"new":[195],"direction":[196],"approaching":[198],"leverages":[202],"naturally-occurring":[204],"audio-visual":[205],"power":[211]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":5}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}