{"id":"https://openalex.org/W7163364006","doi":"https://doi.org/10.48550/arxiv.2606.03672","title":"Foley-Omni: A Unified Multimodal Generation Model from Task-Level Audio Synthesis to Complete Video Soundtrack Generation","display_name":"Foley-Omni: A Unified Multimodal Generation Model from Task-Level Audio Synthesis to Complete Video Soundtrack Generation","publication_year":2026,"publication_date":"2026-06-02","ids":{"openalex":"https://openalex.org/W7163364006","doi":"https://doi.org/10.48550/arxiv.2606.03672"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.03672","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.03672","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.03672","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137788512","display_name":"Ye Tao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tao, Ye","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056518775","display_name":"Lupeng Liu","orcid":"https://orcid.org/0000-0003-2517-7593"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Lupeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137800722","display_name":"Xuenan Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Xuenan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083146027","display_name":"Jiasun Feng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feng, Jiasun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137724812","display_name":"Jiarui Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jiarui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137776854","display_name":"Ying Qin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qin, Ying","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083630691","display_name":"Shuiyang Mao","orcid":"https://orcid.org/0000-0002-9302-185X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mao, Shuiyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137746318","display_name":"Wei Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137724550","display_name":"Shuai Wang (109515)","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Shuai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.6607000231742859,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.6607000231742859,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.09769999980926514,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.046799998730421066,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.4810999929904938},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.44359999895095825},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.44279998540878296},{"id":"https://openalex.org/keywords/audio-signal-processing","display_name":"Audio signal processing","score":0.38530001044273376},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.37770000100135803},{"id":"https://openalex.org/keywords/sound-quality","display_name":"Sound quality","score":0.36399999260902405},{"id":"https://openalex.org/keywords/acoustic-model","display_name":"Acoustic model","score":0.33739998936653137},{"id":"https://openalex.org/keywords/surround-sound","display_name":"Surround sound","score":0.33379998803138733}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.835099995136261},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.5009999871253967},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.48730000853538513},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.4810999929904938},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.44359999895095825},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.44279998540878296},{"id":"https://openalex.org/C127220857","wikidata":"https://www.wikidata.org/wiki/Q2719318","display_name":"Audio signal processing","level":4,"score":0.38530001044273376},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.37770000100135803},{"id":"https://openalex.org/C167310288","wikidata":"https://www.wikidata.org/wiki/Q7564808","display_name":"Sound quality","level":2,"score":0.36399999260902405},{"id":"https://openalex.org/C155635449","wikidata":"https://www.wikidata.org/wiki/Q4674699","display_name":"Acoustic model","level":3,"score":0.33739998936653137},{"id":"https://openalex.org/C2780544925","wikidata":"https://www.wikidata.org/wiki/Q569874","display_name":"Surround sound","level":3,"score":0.33379998803138733},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32839998602867126},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3215000033378601},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3192000091075897},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.3037000000476837},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.29409998655319214},{"id":"https://openalex.org/C64922751","wikidata":"https://www.wikidata.org/wiki/Q4650799","display_name":"Audio signal","level":3,"score":0.2937999963760376},{"id":"https://openalex.org/C3018907156","wikidata":"https://www.wikidata.org/wiki/Q7785058","display_name":"Third generation","level":2,"score":0.28690001368522644},{"id":"https://openalex.org/C128422554","wikidata":"https://www.wikidata.org/wiki/Q20077126","display_name":"Sound recording and reproduction","level":2,"score":0.2858999967575073},{"id":"https://openalex.org/C14999030","wikidata":"https://www.wikidata.org/wiki/Q16346","display_name":"Speech synthesis","level":2,"score":0.2824000120162964},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.2773999869823456},{"id":"https://openalex.org/C157968479","wikidata":"https://www.wikidata.org/wiki/Q3079876","display_name":"Audio mining","level":4,"score":0.26460000872612},{"id":"https://openalex.org/C108803254","wikidata":"https://www.wikidata.org/wiki/Q857512","display_name":"Smacker video","level":4,"score":0.2621999979019165},{"id":"https://openalex.org/C2777655017","wikidata":"https://www.wikidata.org/wiki/Q1501161","display_name":"Toolbox","level":2,"score":0.2597000002861023}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.03672","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.03672","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.03672","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.03672","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.4229370653629303,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"unified":[1,52],"audio":[2,36,54],"generation":[3,55,66,79,103],"models":[4],"can":[5],"support":[6,82],"diverse":[7],"tasks":[8],"across":[9],"speech,":[10,70],"sound":[11,71],"effects,":[12,72],"and":[13,42,73,84,94,125],"music,":[14],"but":[15],"most":[16],"of":[17,33],"them":[18],"still":[19],"focus":[20],"on":[21,115],"isolated":[22,59],"task-level":[23,60],"synthesis.":[24],"However,":[25],"real":[26],"video":[27,64,101],"production":[28],"often":[29],"requires":[30],"multiple":[31],"components":[32],"a":[34,51,76,97],"complete":[35,63],"track":[37],"to":[38,62],"be":[39],"generated":[40],"jointly":[41,68],"consistently":[43],"for":[44,99,128],"the":[45],"same":[46],"video.":[47],"We":[48],"present":[49],"Foley-Omni,":[50],"multimodal":[53],"model":[56],"that":[57,107],"extends":[58],"synthesis":[61,117],"soundtrack":[65,102,130],"by":[67],"modeling":[69],"music":[74],"within":[75],"shared":[77],"latent":[78],"process.":[80],"To":[81],"training":[83],"reproducible":[85],"evaluation,":[86],"we":[87],"develop":[88],"an":[89],"audiovisual":[90,123],"data":[91],"curation":[92],"pipeline":[93],"introduce":[95],"V2ST-Bench,":[96],"benchmark":[98],"holistic":[100],"evaluation.":[104],"Experiments":[105],"show":[106],"Foley-Omni":[108],"achieves":[109],"competitive":[110],"performance":[111],"with":[112],"expert":[113],"systems":[114],"individual":[116],"tasks,":[118],"while":[119],"improving":[120],"speech":[121],"intelligibility,":[122],"consistency":[124],"perceptual":[126],"quality":[127],"mixed":[129],"generation.":[131]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-04T00:00:00"}