{"id":"https://openalex.org/W7160291278","doi":"https://doi.org/10.1145/3811304","title":"UniVidX: A Unified Multimodal Framework for Versatile Video Generation via Diffusion Priors","display_name":"UniVidX: A Unified Multimodal Framework for Versatile Video Generation via Diffusion Priors","publication_year":2026,"publication_date":"2026-07-03","ids":{"openalex":"https://openalex.org/W7160291278","doi":"https://doi.org/10.1145/3811304"},"language":"en","primary_location":{"id":"doi:10.1145/3811304","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3811304","pdf_url":null,"source":{"id":"https://openalex.org/S185367456","display_name":"ACM Transactions on Graphics","issn_l":"0730-0301","issn":["0730-0301","1557-7368"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Graphics","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3811304","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063068906","display_name":"Houyuan Chen","orcid":"https://orcid.org/0009-0009-8191-7550"},"institutions":[{"id":"https://openalex.org/I4210137146","display_name":"Hong Kong Academy of Medicine","ror":"https://ror.org/036b4vb45","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210137146"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Houyuan Chen","raw_affiliation_strings":["MMLAB, HKUST, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0009-0005-4693-2326","affiliations":[{"raw_affiliation_string":"MMLAB, HKUST, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I4210137146","https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135346932","display_name":"Hong Li","orcid":null},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hong Li","raw_affiliation_strings":["Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-4240-3073","affiliations":[{"raw_affiliation_string":"Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135355686","display_name":"Xianghao Kong","orcid":null},"institutions":[{"id":"https://openalex.org/I4210137146","display_name":"Hong Kong Academy of Medicine","ror":"https://ror.org/036b4vb45","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210137146"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Xianghao Kong","raw_affiliation_strings":["MMLAB, HKUST, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0009-0004-9865-4105","affiliations":[{"raw_affiliation_string":"MMLAB, HKUST, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I4210137146","https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135343823","display_name":"Tianrui Zhu","orcid":null},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianrui Zhu","raw_affiliation_strings":["Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0009-0002-5146-019X","affiliations":[{"raw_affiliation_string":"Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135353795","display_name":"Shaocong Xu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shaocong Xu","raw_affiliation_strings":["Beijing Academy of Artificial Intelligence, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-7525-0790","affiliations":[{"raw_affiliation_string":"Beijing Academy of Artificial Intelligence, Beijing, China","institution_ids":["https://openalex.org/I4210100255"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135167345","display_name":"Weiqing Xiao","orcid":null},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weiqing Xiao","raw_affiliation_strings":["Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0009-0003-2548-0485","affiliations":[{"raw_affiliation_string":"Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135407784","display_name":"Yuwei Guo","orcid":null},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yuwei Guo","raw_affiliation_strings":["MMLAB, The Chinese University of Hong Kong, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0009-0003-1516-4083","affiliations":[{"raw_affiliation_string":"MMLAB, The Chinese University of Hong Kong, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135348503","display_name":"Chongjie Ye","orcid":null},"institutions":[{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chongjie Ye","raw_affiliation_strings":["The Chinese University of Hong Kong, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-7123-0220","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Shenzhen, China","institution_ids":["https://openalex.org/I4210116924"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079385579","display_name":"Lvmin Zhang","orcid":"https://orcid.org/0000-0003-3503-5791"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lvmin Zhang","raw_affiliation_strings":["Stanford University, Stanford, USA"],"raw_orcid":"https://orcid.org/0000-0003-3503-5791","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135331182","display_name":"Hao Zhao","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hao Zhao","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-7903-581X","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5135360571","display_name":"Anyi Rao","orcid":null},"institutions":[{"id":"https://openalex.org/I4210137146","display_name":"Hong Kong Academy of Medicine","ror":"https://ror.org/036b4vb45","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210137146"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Anyi Rao","raw_affiliation_strings":["MMLAB, HKUST, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0003-1004-7753","affiliations":[{"raw_affiliation_string":"MMLAB, HKUST, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I4210137146","https://openalex.org/I889458895"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":9,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.54258028,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"45","issue":"4","first_page":"1","last_page":"17"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9495999813079834,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9495999813079834,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.008999999612569809,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11019","display_name":"Image Enhancement Techniques","score":0.00800000037997961,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/prior-probability","display_name":"Prior probability","score":0.7141000032424927},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5180000066757202},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.45730000734329224},{"id":"https://openalex.org/keywords/discriminative-model","display_name":"Discriminative model","score":0.43389999866485596},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.4171999990940094},{"id":"https://openalex.org/keywords/rgb-color-model","display_name":"RGB color model","score":0.38760000467300415},{"id":"https://openalex.org/keywords/masking","display_name":"Masking (illustration)","score":0.3625999987125397},{"id":"https://openalex.org/keywords/conditional-random-field","display_name":"Conditional random field","score":0.33719998598098755}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7703999876976013},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.7141000032424927},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5778999924659729},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5180000066757202},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.45730000734329224},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.43389999866485596},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.41839998960494995},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.4171999990940094},{"id":"https://openalex.org/C82990744","wikidata":"https://www.wikidata.org/wiki/Q166194","display_name":"RGB color model","level":2,"score":0.38760000467300415},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37720000743865967},{"id":"https://openalex.org/C2777402240","wikidata":"https://www.wikidata.org/wiki/Q6783436","display_name":"Masking (illustration)","level":2,"score":0.3625999987125397},{"id":"https://openalex.org/C152565575","wikidata":"https://www.wikidata.org/wiki/Q1124538","display_name":"Conditional random field","level":2,"score":0.33719998598098755},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.3314000070095062},{"id":"https://openalex.org/C77660652","wikidata":"https://www.wikidata.org/wiki/Q150971","display_name":"Computer graphics","level":2,"score":0.28360000252723694},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.28189998865127563},{"id":"https://openalex.org/C21442007","wikidata":"https://www.wikidata.org/wiki/Q1027879","display_name":"Graphics","level":2,"score":0.2808000147342682},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.27320000529289246},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2728999853134155},{"id":"https://openalex.org/C61224824","wikidata":"https://www.wikidata.org/wiki/Q2260434","display_name":"Mixture model","level":2,"score":0.27079999446868896},{"id":"https://openalex.org/C2776449333","wikidata":"https://www.wikidata.org/wiki/Q7928781","display_name":"View synthesis","level":3,"score":0.26100000739097595},{"id":"https://openalex.org/C43555835","wikidata":"https://www.wikidata.org/wiki/Q2300258","display_name":"Conditional probability distribution","level":2,"score":0.2606000006198883},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.25839999318122864}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3811304","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3811304","pdf_url":null,"source":{"id":"https://openalex.org/S185367456","display_name":"ACM Transactions on Graphics","issn_l":"0730-0301","issn":["0730-0301","1557-7368"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Graphics","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2605.00658","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2605.00658","pdf_url":"https://arxiv.org/pdf/2605.00658","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3811304","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3811304","pdf_url":null,"source":{"id":"https://openalex.org/S185367456","display_name":"ACM Transactions on Graphics","issn_l":"0730-0301","issn":["0730-0301","1557-7368"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Graphics","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure","score":0.4025461673736572}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"progress":[1],"has":[2],"shown":[3],"that":[4,223],"video":[5,65],"diffusion":[6],"models":[7,24,33,225],"(VDMs)":[8],"can":[9],"be":[10],"repurposed":[11],"to":[12,58,62,70,88,131],"solve":[13],"various":[14],"multimodal":[15,55,84],"graphics":[16],"tasks.":[17,234],"However,":[18],"existing":[19],"approaches":[20],"predominantly":[21],"train":[22],"separate":[23],"for":[25,198,211],"each":[26],"specific":[27],"problem":[28],"setting.":[29],"This":[30],"practice":[31],"locks":[32],"into":[34,119],"fixed":[35,138],"input-output":[36],"mappings,":[37],"and":[38,97,122,149,183,201,208,215],"typically":[39],"ignores":[40],"the":[41,93,129,162],"joint":[42],"correlations":[43],"across":[44,174,232],"modalities.":[45],"In":[46],"this":[47],"paper,":[48],"we":[49,105,127,145,170],"present":[50],"UniVidX":[51],",":[52],"a":[53,153,157],"unified":[54],"framework":[56,189],"designed":[57],"leverage":[59],"VDM":[60],"priors":[61],"enable":[63,128],"versatile":[64],"generation.":[66],"Our":[67],"goal":[68],"is":[69],"(i)":[71],"master":[72],"diverse":[73],"pixel-aligned":[74],"tasks":[75],"by":[76,115,190],"formulating":[77],"them":[78,151],"as":[79,156],"conditional":[80,134],"generation":[81,135,158],"problems":[82],"within":[83],"space,":[85],"(ii)":[86],"adapt":[87],"modality-specific":[89,178],"distributions":[90],"without":[91],"compromising":[92],"backbone's":[94],"native":[95],"priors,":[96],"(iii)":[98],"ensure":[99],"cross-modal":[100],"consistency":[101],"during":[102,125],"synthesis.":[103],"Concretely,":[104],"propose":[106],"three":[107],"key":[108],"designs:":[109],"1)":[110,196],"Stochastic":[111],"Condition":[112],"Masking":[113],"(SCM):":[114],"randomly":[116],"partitioning":[117],"modalities":[118,175],"clean":[120],"conditions":[121],"noisy":[123],"targets":[124],"training,":[126],"model":[130],"learn":[132],"omni-directional":[133],"rather":[136],"than":[137,252],"mappings.":[139],"2)":[140,209],"Decoupled":[141],"Gated":[142],"LoRA":[143],"(DGL):":[144],"attach":[146],"per-modality":[147],"LoRAs":[148],"activate":[150],"when":[152,245],"modality":[154],"serves":[155],"target,":[159],"thereby":[160],"preserving":[161],"VDM's":[163],"strong":[164],"priors.":[165],"3)":[166],"Cross-Modal":[167],"Self-Attention":[168],"(CMSA):":[169],"explicitly":[171],"share":[172],"keys/values":[173],"while":[176],"maintaining":[177],"queries,":[179],"facilitating":[180],"information":[181],"exchange":[182],"inter-modal":[184],"alignment.":[185],"We":[186],"validate":[187],"our":[188],"instantiating":[191],"it":[192],"in":[193,241],"two":[194],"domains:":[195],"UniVid-Intrinsic":[197],"RGB":[199,213],"videos":[200,214],"their":[202,216],"intrinsic":[203],"maps":[204],"(albedo,":[205],"irradiance,":[206],"normal),":[207],"UniVid-Alpha":[210],"blended":[212],"constituent":[217],"RGBA":[218],"layers.":[219],"Experimental":[220],"results":[221],"demonstrate":[222],"both":[224],"achieve":[226],"performance":[227],"competitive":[228],"with":[229],"state-of-the-art":[230],"methods":[231],"distinct":[233],"Notably,":[235],"they":[236],"exhibit":[237],"robust":[238],"generalization":[239],"capabilities":[240],"in-the-wild":[242],"scenarios,":[243],"even":[244],"trained":[246],"on":[247],"limited":[248],"datasets":[249],"of":[250],"fewer":[251],"1k":[253],"videos.":[254]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-06T00:00:00"}