{"id":"https://openalex.org/W7151394433","doi":"https://doi.org/10.48550/arxiv.2604.04348","title":"OmniSonic: Towards Universal and Holistic Audio Generation from Video and Text","display_name":"OmniSonic: Towards Universal and Holistic Audio Generation from Video and Text","publication_year":2026,"publication_date":"2026-04-06","ids":{"openalex":"https://openalex.org/W7151394433","doi":"https://doi.org/10.48550/arxiv.2604.04348"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.04348","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.04348","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.04348","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011256066","display_name":"Weiguo Pian","orcid":"https://orcid.org/0000-0002-4653-7415"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pian, Weiguo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133118253","display_name":"Saksham Singh Kushwaha","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kushwaha, Saksham Singh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064079987","display_name":"Zhimin Chen","orcid":"https://orcid.org/0009-0001-8035-4807"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Zhimin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028337614","display_name":"Shijian Deng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Deng, Shijian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133140841","display_name":"Kai Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Kai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133097556","display_name":"Yunhui Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Yunhui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133067338","display_name":"Yapeng Tian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tian, Yapeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.3190000057220459,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.3190000057220459,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.1835000067949295,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.10859999805688858,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.5128999948501587},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4927000105381012},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.49000000953674316},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4871000051498413},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.47540000081062317},{"id":"https://openalex.org/keywords/spectrogram","display_name":"Spectrogram","score":0.4316999912261963},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.4099999964237213},{"id":"https://openalex.org/keywords/soundscape","display_name":"Soundscape","score":0.39340001344680786}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7799000144004822},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.5128999948501587},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4927000105381012},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.49000000953674316},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4871000051498413},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.47540000081062317},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.47130000591278076},{"id":"https://openalex.org/C45273575","wikidata":"https://www.wikidata.org/wiki/Q578970","display_name":"Spectrogram","level":2,"score":0.4316999912261963},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.4099999964237213},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4077000021934509},{"id":"https://openalex.org/C142795923","wikidata":"https://www.wikidata.org/wiki/Q1358257","display_name":"Soundscape","level":3,"score":0.39340001344680786},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.35569998621940613},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3555999994277954},{"id":"https://openalex.org/C113364801","wikidata":"https://www.wikidata.org/wiki/Q26674","display_name":"High fidelity","level":2,"score":0.3499000072479248},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.3400000035762787},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.329800009727478},{"id":"https://openalex.org/C91607612","wikidata":"https://www.wikidata.org/wiki/Q1416058","display_name":"Sonification","level":2,"score":0.29179999232292175},{"id":"https://openalex.org/C64922751","wikidata":"https://www.wikidata.org/wiki/Q4650799","display_name":"Audio signal","level":3,"score":0.29170000553131104},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.289900004863739},{"id":"https://openalex.org/C135343436","wikidata":"https://www.wikidata.org/wiki/Q170406","display_name":"Rhythm","level":2,"score":0.28209999203681946},{"id":"https://openalex.org/C100142294","wikidata":"https://www.wikidata.org/wiki/Q2070426","display_name":"Selective auditory attention","level":4,"score":0.2685000002384186},{"id":"https://openalex.org/C38129911","wikidata":"https://www.wikidata.org/wiki/Q4820038","display_name":"Auditory scene analysis","level":3,"score":0.26739999651908875},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.2540999948978424}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.04348","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.04348","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.04348","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.04348","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"In":[0],"this":[1],"paper,":[2],"we":[3,95,144],"propose":[4],"Universal":[5],"Holistic":[6],"Audio":[7],"Generation":[8],"(UniHAGen),":[9],"a":[10,98,110,131,147,178],"task":[11],"for":[12,181],"synthesizing":[13],"comprehensive":[14],"auditory":[15,55,67],"scenes":[16,68],"that":[17,47,113,136,164],"include":[18],"both":[19,70,171],"on-screen":[20,44,120],"and":[21,32,72,106,126,174,183],"off-screen":[22,54,73,123],"sounds":[23,46],"across":[24],"diverse":[25],"domains":[26],"(e.g.,":[27],"ambient":[28],"events,":[29,52],"musical":[30],"instruments,":[31],"human":[33,89,175],"speech).":[34],"Prior":[35],"video-conditioned":[36],"audio":[37,185],"generation":[38,62],"models":[39,63],"typically":[40],"focus":[41],"on":[42,104,170],"producing":[43],"environmental":[45,121,124],"correspond":[48],"to":[49,65,79,85,118],"visible":[50],"sounding":[51],"neglecting":[53],"events.":[56],"While":[57],"recent":[58],"holistic":[59,184],"joint":[60],"text-video-to-audio":[61],"aim":[64],"produce":[66],"with":[69,130,150],"on-":[71],"sound":[74],"but":[75],"they":[76],"are":[77],"limited":[78],"non-speech":[80],"sounds,":[81],"lacking":[82],"the":[83],"ability":[84],"generate":[86],"or":[87],"integrate":[88],"speech.":[90],"To":[91],"overcome":[92],"these":[93],"limitations,":[94],"introduce":[96],"OmniSonic,":[97],"flow-matching-based":[99],"diffusion":[100],"framework":[101],"jointly":[102],"conditioned":[103],"video":[105],"text.":[107],"It":[108],"features":[109],"TriAttn-DiT":[111],"architecture":[112],"performs":[114],"three":[115,156],"cross-attention":[116],"operations":[117],"process":[119],"sound,":[122,125],"speech":[127],"conditions":[128],"simultaneously,":[129],"Mixture-of-Experts":[132],"(MoE)":[133],"gating":[134],"mechanism":[135],"adaptively":[137],"balances":[138],"their":[139],"contributions":[140],"during":[141],"generation.":[142,186],"Furthermore,":[143],"construct":[145],"UniHAGen-Bench,":[146],"new":[148],"benchmark":[149],"over":[151],"one":[152],"thousand":[153],"samples":[154],"covering":[155],"representative":[157],"on/off-screen":[158],"speech-environment":[159],"scenarios.":[160],"Extensive":[161],"experiments":[162],"show":[163],"OmniSonic":[165],"consistently":[166],"outperforms":[167],"state-of-the-art":[168],"approaches":[169],"objective":[172],"metrics":[173],"evaluations,":[176],"establishing":[177],"strong":[179],"baseline":[180],"universal":[182],"Project":[187],"page:":[188],"https://weiguopian.github.io/OmniSonic_webpage/":[189]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-08T00:00:00"}