{"id":"https://openalex.org/W7168500566","doi":"https://doi.org/10.1145/3799902.3811191","title":"Audio-Omni: Extending Multi-modal Understanding to Versatile Audio Generation and Editing","display_name":"Audio-Omni: Extending Multi-modal Understanding to Versatile Audio Generation and Editing","publication_year":2026,"publication_date":"2026-07-16","ids":{"openalex":"https://openalex.org/W7168500566","doi":"https://doi.org/10.1145/3799902.3811191"},"language":null,"primary_location":{"id":"doi:10.1145/3799902.3811191","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3799902.3811191","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Papers","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3799902.3811191","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5141044553","display_name":"Zeyue Tian","orcid":"https://orcid.org/0000-0002-7278-3708"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zeyue Tian","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0002-7278-3708","affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013134358","display_name":"Binxin Yang","orcid":"https://orcid.org/0000-0003-4110-1986"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Binxin Yang","raw_affiliation_strings":["Tencent, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0003-4110-1986","affiliations":[{"raw_affiliation_string":"Tencent, Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073859765","display_name":"Z T Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhaoyang Liu","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0003-0258-3097","affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051262853","display_name":"J. Zhang","orcid":"https://orcid.org/0009-0008-0999-1046"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiexuan Zhang","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0006-8609-8919","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141007118","display_name":"Ruibin Yuan","orcid":"https://orcid.org/0009-0002-0539-6916"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Ruibin Yuan","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0009-0002-0539-6916","affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102630122","display_name":"Hubery Yin","orcid":null},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hubery Yin","raw_affiliation_strings":["Tencent, Shenzhen, China"],"raw_orcid":"https://orcid.org/0009-0008-9492-8729","affiliations":[{"raw_affiliation_string":"Tencent, Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100719529","display_name":"Qifeng Chen","orcid":"https://orcid.org/0000-0003-2199-3948"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Qifeng Chen","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0003-2199-3948","affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100369821","display_name":"Chen Li","orcid":"https://orcid.org/0000-0002-2450-8525"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chen Li","raw_affiliation_strings":["Tencent, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-2450-8525","affiliations":[{"raw_affiliation_string":"Tencent, Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002435489","display_name":"Jing Lyu","orcid":"https://orcid.org/0000-0002-2261-317X"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jing Lyu","raw_affiliation_strings":["Tencent, Shenzhen, China"],"raw_orcid":"https://orcid.org/0009-0004-2021-0256","affiliations":[{"raw_affiliation_string":"Tencent, Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100652844","display_name":"Wei Xue","orcid":"https://orcid.org/0000-0002-4942-7748"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Wei Xue","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0002-4942-7748","affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5138131714","display_name":"Yike Guo","orcid":"https://orcid.org/0009-0005-8401-282X"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yike Guo","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0009-0005-8401-282X","affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I200769079"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.6712999939918518},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.4814999997615814},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.4471000134944916},{"id":"https://openalex.org/keywords/image-editing","display_name":"Image editing","score":0.39469999074935913},{"id":"https://openalex.org/keywords/architecture","display_name":"Architecture","score":0.36660000681877136},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.3564000129699707}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8422999978065491},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.6712999939918518},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.4814999997615814},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.4471000134944916},{"id":"https://openalex.org/C2776674983","wikidata":"https://www.wikidata.org/wiki/Q545981","display_name":"Image editing","level":3,"score":0.39469999074935913},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.36660000681877136},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3564000129699707},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3495999872684479},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.31949999928474426},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3070000112056732},{"id":"https://openalex.org/C2777655017","wikidata":"https://www.wikidata.org/wiki/Q1501161","display_name":"Toolbox","level":2,"score":0.2734000086784363},{"id":"https://openalex.org/C14999030","wikidata":"https://www.wikidata.org/wiki/Q16346","display_name":"Speech synthesis","level":2,"score":0.2524000108242035}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3799902.3811191","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3799902.3811191","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Papers","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3799902.3811191","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3799902.3811191","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Papers","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W2052666245","https://openalex.org/W2771361008","https://openalex.org/W3015371781","https://openalex.org/W3094550259","https://openalex.org/W4372266552","https://openalex.org/W4387969125","https://openalex.org/W4392903114","https://openalex.org/W4392904679","https://openalex.org/W4393178509","https://openalex.org/W4401043564","https://openalex.org/W4402670276","https://openalex.org/W4402672034","https://openalex.org/W4402727574","https://openalex.org/W4402727849","https://openalex.org/W4403780831","https://openalex.org/W4412945617","https://openalex.org/W4413145987","https://openalex.org/W4413146212","https://openalex.org/W4413147805","https://openalex.org/W4413155808","https://openalex.org/W4415538091","https://openalex.org/W7133224691","https://openalex.org/W7133229781"],"related_works":[],"abstract_inverted_index":{"Recent":[0],"progress":[1],"in":[2,9,111],"multimodal":[3],"models":[4],"has":[5],"spurred":[6],"rapid":[7],"advances":[8],"audio":[10,47,112,178,187],"understanding,":[11],"generation,":[12,50,170,172,179],"and":[13,49,71,77,173,192],"editing.":[14],"However,":[15],"these":[16],"capabilities":[17],"are":[18],"typically":[19],"addressed":[20],"by":[21],"specialized":[22,155],"models,":[23],"leaving":[24],"the":[25,64,107],"development":[26],"of":[27,140],"a":[28,88,98,117,138,181],"truly":[29],"unified":[30,144],"framework":[31,67],"that":[32,132],"can":[33],"seamlessly":[34],"integrate":[35],"all":[36],"three":[37],"tasks":[38],"underexplored.":[39],"While":[40],"some":[41],"pioneering":[42],"works":[43],"have":[44],"explored":[45],"unifying":[46],"understanding":[48,83],"they":[51],"often":[52],"remain":[53],"confined":[54],"to":[55,68,154],"specific":[56],"domains.":[57],"To":[58,105],"address":[59],"this,":[60],"we":[61,114],"introduce":[62],"Audio-Omni,":[63],"first":[65],"end-to-end":[66],"unify":[69],"generation":[70],"editing":[72,127],"across":[73,137],"general":[74],"sound,":[75],"music,":[76],"speech":[78],"domains,":[79],"with":[80,97,151],"integrated":[81],"multi-modal":[82],"capabilities.":[84],"Our":[85],"architecture":[86],"synergizes":[87],"frozen":[89],"Multimodal":[90],"Large":[91],"Language":[92],"Model":[93],"for":[94,102,177],"high-level":[95],"reasoning":[96,169],"trainable":[99],"Diffusion":[100],"Transformer":[101],"high-fidelity":[103],"synthesis.":[104],"overcome":[106],"critical":[108],"data":[109],"scarcity":[110],"editing,":[113],"construct":[115],"AudioEdit,":[116],"new":[118],"large-scale":[119],"dataset":[120,193],"comprising":[121],"over":[122],"one":[123],"million":[124],"meticulously":[125],"curated":[126],"pairs.":[128],"Extensive":[129],"experiments":[130],"demonstrate":[131],"Audio-Omni":[133,162],"achieves":[134],"state-of-the-art":[135],"performance":[136,148],"suite":[139],"benchmarks,":[141],"outperforming":[142],"prior":[143],"approaches":[145],"while":[146],"achieving":[147],"on":[149,198],"par":[150],"or":[152],"superior":[153],"expert":[156],"models.":[157],"Beyond":[158],"its":[159],"core":[160],"capabilities,":[161,166],"exhibits":[163],"remarkable":[164],"inherited":[165],"including":[167],"knowledge-augmented":[168],"in-context":[171],"zero-shot":[174],"cross-lingual":[175],"control":[176],"highlighting":[180],"promising":[182],"direction":[183],"toward":[184],"universal":[185],"generative":[186],"intelligence.":[188],"The":[189],"code,":[190],"model,":[191],"will":[194],"be":[195],"publicly":[196],"released":[197],"https://zeyuet.github.io/Audio-Omni.":[199]},"counts_by_year":[],"updated_date":"2026-07-17T05:58:32.477365","created_date":"2026-07-17T00:00:00"}