{"id":"https://openalex.org/W7161008418","doi":"https://doi.org/10.48550/arxiv.2605.12480","title":"OmniNFT: Modality-wise Omni Diffusion Reinforcement for Joint Audio-Video Generation","display_name":"OmniNFT: Modality-wise Omni Diffusion Reinforcement for Joint Audio-Video Generation","publication_year":2026,"publication_date":"2026-05-12","ids":{"openalex":"https://openalex.org/W7161008418","doi":"https://doi.org/10.48550/arxiv.2605.12480"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.12480","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12480","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.12480","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136083384","display_name":"Guohui Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Guohui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136007840","display_name":"XiaoXiao Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, XiaoXiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136083527","display_name":"Jie Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Jie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136047926","display_name":"Hang xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Hang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103662468","display_name":"Hu Yu","orcid":"https://orcid.org/0000-0003-0598-8989"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Hu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083298839","display_name":"Siming Fu","orcid":"https://orcid.org/0000-0003-3257-1011"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Siming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136059593","display_name":"Yuming Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yuming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040083129","display_name":"Zeyue Xue","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xue, Zeyue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136058968","display_name":"Lin Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Lin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020769158","display_name":"Haoyang Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Haoyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136059222","display_name":"Nan Duan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Duan, Nan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136070636","display_name":"Feng Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Feng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.4632999897003174,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.4632999897003174,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.2354000061750412,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.0478999987244606,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/joint","display_name":"Joint (building)","score":0.6305000185966492},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6128000020980835},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.538100004196167},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.4422999918460846},{"id":"https://openalex.org/keywords/diffusion","display_name":"Diffusion","score":0.3718000054359436},{"id":"https://openalex.org/keywords/mode","display_name":"Mode (computer interface)","score":0.353300005197525},{"id":"https://openalex.org/keywords/extension","display_name":"Extension (predicate logic)","score":0.3515999913215637}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7196000218391418},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.6305000185966492},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6128000020980835},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.538100004196167},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.4422999918460846},{"id":"https://openalex.org/C69357855","wikidata":"https://www.wikidata.org/wiki/Q163214","display_name":"Diffusion","level":2,"score":0.3718000054359436},{"id":"https://openalex.org/C48677424","wikidata":"https://www.wikidata.org/wiki/Q6888088","display_name":"Mode (computer interface)","level":2,"score":0.353300005197525},{"id":"https://openalex.org/C2778029271","wikidata":"https://www.wikidata.org/wiki/Q5421931","display_name":"Extension (predicate logic)","level":2,"score":0.3515999913215637},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.3398999869823456},{"id":"https://openalex.org/C47798520","wikidata":"https://www.wikidata.org/wiki/Q190157","display_name":"Transmitter","level":3,"score":0.33799999952316284},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.32710000872612},{"id":"https://openalex.org/C3018907156","wikidata":"https://www.wikidata.org/wiki/Q7785058","display_name":"Third generation","level":2,"score":0.29910001158714294},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.2939999997615814},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.2924000024795532},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2912999987602234},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.26759999990463257},{"id":"https://openalex.org/C186886427","wikidata":"https://www.wikidata.org/wiki/Q5441213","display_name":"Feedback loop","level":2,"score":0.25110000371932983}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.12480","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12480","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.12480","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12480","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"advances":[1],"in":[2,53,209],"joint":[3,35],"audio-video":[4,36,188,218],"generation":[5,37,153],"have":[6],"been":[7],"remarkable,":[8],"yet":[9],"real-world":[10],"applications":[11],"demand":[12],"strong":[13],"per-modality":[14],"fidelity,":[15],"cross-modal":[16,96,172,215],"alignment,":[17,216],"and":[18,33,190,197,211,217],"fine-grained":[19,95,191],"synchronization.":[20,219],"Reinforcement":[21],"Learning":[22],"(RL)":[23],"offers":[24],"a":[25,72,113,129],"promising":[26],"paradigm,":[27],"but":[28],"its":[29],"extension":[30],"to":[31,50,100,119,149,187],"multi-objective":[32,58],"multi-modal":[34,75],"remains":[38],"unexplored.":[39],"Notably,":[40],"our":[41],"in-depth":[42],"analysis":[43],"first":[44],"reveals":[45],"that":[46,107,204],"the":[47,62,200],"primary":[48],"obstacles":[49],"applying":[51],"RL":[52,109,134],"this":[54],"stem":[55],"from:":[56],"(i)":[57],"advantages":[59,63,148],"inconsistency,":[60],"where":[61,78,94],"of":[64],"multimodal":[65],"outputs":[66],"are":[67],"not":[68],"always":[69],"consistent":[70],"within":[71],"group;":[73],"(ii)":[74],"gradients":[76,80,163],"imbalance,":[77],"video-branch":[79,162],"leak":[81],"into":[82],"shallow":[83,165],"audio":[84,166,210],"layers":[85,167],"responsible":[86],"for":[87,171],"intra-modal":[88],"generation;":[89],"(iii)":[90],"uniform":[91],"credit":[92],"assignment,":[93],"alignment":[97],"regions":[98,185],"fail":[99],"get":[101],"efficient":[102],"exploration.":[103],"These":[104],"shortcomings":[105],"suggest":[106],"vanilla":[108],"fine-tuning":[110],"strategy":[111],"with":[112,136,199],"single":[114],"global":[115],"advantage":[116,142],"often":[117],"leads":[118],"suboptimal":[120],"results.":[121],"To":[122],"address":[123],"these":[124],"challenges,":[125],"we":[126],"propose":[127],"OmniNFT,":[128],"novel":[130],"modality-aware":[131],"online":[132],"diffusion":[133],"framework":[135],"three":[137],"key":[138],"innovations:":[139],"(1)":[140],"Modality-wise":[141],"routing,":[143],"which":[144,159,179],"routes":[145],"independent":[146],"per-reward":[147],"their":[150],"respective":[151],"modality":[152],"branches.":[154],"(2)":[155],"Layer-wise":[156],"gradient":[157],"surgery,":[158],"selectively":[160],"detaches":[161],"on":[164,195],"while":[168],"retaining":[169],"those":[170],"interaction":[173],"layers.":[174],"(3)":[175],"Region-wise":[176],"loss":[177],"reweighting,":[178],"modulates":[180],"policy":[181],"optimization":[182],"toward":[183],"critical":[184],"related":[186],"synchronization":[189],"alignment.":[192],"Extensive":[193],"experiments":[194],"JavisBench":[196],"VBench":[198],"LTX-2":[201],"backbone":[202],"demonstrate":[203],"OmniNFT":[205],"achieves":[206],"comprehensive":[207],"improvements":[208],"video":[212],"perceptual":[213],"quality,":[214]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-14T00:00:00"}