{"id":"https://openalex.org/W4362514612","doi":"https://doi.org/10.1109/tmm.2023.3262180","title":"Sounding Video Generator: A Unified Framework for Text-Guided Sounding Video Generation","display_name":"Sounding Video Generator: A Unified Framework for Text-Guided Sounding Video Generation","publication_year":2023,"publication_date":"2023-04-03","ids":{"openalex":"https://openalex.org/W4362514612","doi":"https://doi.org/10.1109/tmm.2023.3262180"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2023.3262180","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2023.3262180","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100320732","display_name":"Jia-Wei Liu","orcid":"https://orcid.org/0000-0001-5438-2524"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiawei Liu","raw_affiliation_strings":["The Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","School of Artificial Intelligence, University of Chinese Academy of Sciences, China"],"raw_orcid":"https://orcid.org/0000-0001-5438-2524","affiliations":[{"raw_affiliation_string":"The Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210094879"]},{"raw_affiliation_string":"School of Artificial Intelligence, University of Chinese Academy of Sciences, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100683352","display_name":"Weining Wang","orcid":"https://orcid.org/0000-0001-7299-6431"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weining Wang","raw_affiliation_strings":["The Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-7299-6431","affiliations":[{"raw_affiliation_string":"The Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210094879"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101953192","display_name":"Sihan Chen","orcid":"https://orcid.org/0000-0002-9453-6265"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sihan Chen","raw_affiliation_strings":["The Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","School of Artificial Intelligence, University of Chinese Academy of Sciences, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210094879"]},{"raw_affiliation_string":"School of Artificial Intelligence, University of Chinese Academy of Sciences, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007156427","display_name":"Xinxin Zhu","orcid":"https://orcid.org/0000-0002-5070-5194"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xinxin Zhu","raw_affiliation_strings":["The Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210094879"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5108392430","display_name":"Jing Liu","orcid":"https://orcid.org/0000-0003-0903-9131"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jing Liu","raw_affiliation_strings":["The Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","School of Artificial Intelligence, University of Chinese Academy of Sciences, China"],"raw_orcid":"https://orcid.org/0000-0003-0903-9131","affiliations":[{"raw_affiliation_string":"The Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210094879"]},{"raw_affiliation_string":"School of Artificial Intelligence, University of Chinese Academy of Sciences, China","institution_ids":["https://openalex.org/I4210165038"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.9769,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.73186422,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"26","issue":null,"first_page":"141","last_page":"153"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12290","display_name":"Human Motion and Animation","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/depth-sounding","display_name":"Depth sounding","score":0.8075904846191406},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8072324991226196},{"id":"https://openalex.org/keywords/generator","display_name":"Generator (circuit theory)","score":0.5082266330718994},{"id":"https://openalex.org/keywords/power","display_name":"Power (physics)","score":0.1292429268360138},{"id":"https://openalex.org/keywords/geology","display_name":"Geology","score":0.10161513090133667}],"concepts":[{"id":"https://openalex.org/C55510283","wikidata":"https://www.wikidata.org/wiki/Q1382947","display_name":"Depth sounding","level":2,"score":0.8075904846191406},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8072324991226196},{"id":"https://openalex.org/C2780992000","wikidata":"https://www.wikidata.org/wiki/Q17016113","display_name":"Generator (circuit theory)","level":3,"score":0.5082266330718994},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.1292429268360138},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.10161513090133667},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2023.3262180","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2023.3262180","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.5699999928474426,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G1283858192","display_name":null,"funder_award_id":"U21B2043","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1508456709","display_name":null,"funder_award_id":"62102416","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4467203512","display_name":null,"funder_award_id":"62102419","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":68,"referenced_works":["https://openalex.org/W24089286","https://openalex.org/W1710476689","https://openalex.org/W2060300932","https://openalex.org/W2064675550","https://openalex.org/W2108598243","https://openalex.org/W2547411571","https://openalex.org/W2593116425","https://openalex.org/W2618530766","https://openalex.org/W2619947201","https://openalex.org/W2752796333","https://openalex.org/W2832876791","https://openalex.org/W2962784628","https://openalex.org/W2962865004","https://openalex.org/W2962934715","https://openalex.org/W2963092440","https://openalex.org/W2964245526","https://openalex.org/W2964345931","https://openalex.org/W2966421984","https://openalex.org/W2981851635","https://openalex.org/W2984008963","https://openalex.org/W3002552512","https://openalex.org/W3015371781","https://openalex.org/W3025665229","https://openalex.org/W3046890131","https://openalex.org/W3095289378","https://openalex.org/W3108240585","https://openalex.org/W3133428285","https://openalex.org/W3152733922","https://openalex.org/W3157199281","https://openalex.org/W3170427719","https://openalex.org/W3180355996","https://openalex.org/W3204588463","https://openalex.org/W3205475937","https://openalex.org/W3207290297","https://openalex.org/W3214910576","https://openalex.org/W3215495615","https://openalex.org/W4238100585","https://openalex.org/W4281632497","https://openalex.org/W4285058679","https://openalex.org/W4294541506","https://openalex.org/W6600983433","https://openalex.org/W6631190155","https://openalex.org/W6637373629","https://openalex.org/W6726497184","https://openalex.org/W6726983635","https://openalex.org/W6739901393","https://openalex.org/W6745193426","https://openalex.org/W6762931180","https://openalex.org/W6763832098","https://openalex.org/W6765779288","https://openalex.org/W6767111847","https://openalex.org/W6770805772","https://openalex.org/W6773226109","https://openalex.org/W6776700526","https://openalex.org/W6780091408","https://openalex.org/W6780316881","https://openalex.org/W6783867762","https://openalex.org/W6787802714","https://openalex.org/W6790978476","https://openalex.org/W6791353385","https://openalex.org/W6793801364","https://openalex.org/W6794385437","https://openalex.org/W6796242362","https://openalex.org/W6796830075","https://openalex.org/W6802805937","https://openalex.org/W6803344029","https://openalex.org/W6838785959","https://openalex.org/W6955071965"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2809612451","https://openalex.org/W1974079146","https://openalex.org/W2373976954","https://openalex.org/W2616411725","https://openalex.org/W2474955975","https://openalex.org/W2383251271","https://openalex.org/W2386558711"],"abstract_inverted_index":{"As":[0],"a":[1,40,56,84,121,145],"combination":[2],"of":[3,23,44,111,161],"visual":[4,24,74,112,131],"and":[5,49,76,93,96,114,133,179],"audio":[6,27,65,77,115,134,174],"signals,":[7],"video":[8,14,47,142],"is":[9,105,124,151],"inherently":[10],"multi-modal.":[11],"However,":[12],"existing":[13,167],"generation":[15,48,169,175],"methods":[16,170,176],"are":[17,32],"primarily":[18],"intended":[19],"for":[20,59,117,139,153],"the":[21,51,70,98,159],"synthesis":[22],"frames,":[25,132],"whereas":[26],"signals":[28,116,135],"in":[29],"realistic":[30,61],"videos":[31,62],"disregarded.":[33],"In":[34],"this":[35],"work,":[36],"we":[37,68],"concentrate":[38],"on":[39,177],"rarely":[41],"investigated":[42],"problem":[43],"text-guided":[45],"sounding":[46,141],"propose":[50],"Sounding":[52],"Video":[53],"Generator":[54],"(SVG),":[55],"unified":[57],"framework":[58],"generating":[60],"along":[63],"with":[64,166],"signals.":[66],"Specifically,":[67],"present":[69],"SVG-VQGAN":[71,82],"to":[72,90,107,126],"transform":[73],"frames":[75,113],"mel-spectrograms":[78],"into":[79],"discrete":[80],"tokens.":[81],"applies":[83],"novel":[85],"hybrid":[86],"contrastive":[87,118],"learning":[88],"method":[89,163],"model":[91,127],"inter-modal":[92],"intra-modal":[94],"consistency":[95],"improve":[97],"quantized":[99],"representations.":[100],"A":[101],"cross-modal":[102],"attention":[103],"module":[104],"employed":[106],"extract":[108],"associated":[109],"features":[110],"learning.":[119],"Then,":[120],"Transformer-based":[122],"decoder":[123],"used":[125],"associations":[128],"between":[129],"texts,":[130],"at":[136],"token":[137],"level":[138],"auto-regressive":[140],"generation.":[143],"AudioSet-Cap,":[144],"human":[146],"annotated":[147],"text-video-audio":[148],"paired":[149],"dataset,":[150],"produced":[152],"training":[154],"SVG.":[155],"Experimental":[156],"results":[157],"demonstrate":[158],"superiority":[160],"our":[162],"when":[164],"compared":[165],"text-to-video":[168],"as":[171,173],"well":[172],"Kinetics":[178],"VAS":[180],"datasets.":[181]},"counts_by_year":[{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":3}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}