{"id":"https://openalex.org/W7169016546","doi":"https://doi.org/10.48550/arxiv.2607.14088","title":"VideoRAE: Taming Video Foundation Models for Generative Modeling via Representation Autoencoders","display_name":"VideoRAE: Taming Video Foundation Models for Generative Modeling via Representation Autoencoders","publication_year":2026,"publication_date":"2026-07-15","ids":{"openalex":"https://openalex.org/W7169016546","doi":"https://doi.org/10.48550/arxiv.2607.14088"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.14088","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.14088","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.14088","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008526590","display_name":"Zhihao Xie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xie, Zhihao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141040055","display_name":"Junfeng Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Junfeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140984636","display_name":"Xinting Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Xinting","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140994352","display_name":"Junchao Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Junchao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5088052728","display_name":"Lu Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9602000117301941,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9602000117301941,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.012299999594688416,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.004600000102072954,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/autoencoder","display_name":"Autoencoder","score":0.8880000114440918},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.6901999711990356},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.6365000009536743},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.46160000562667847},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.45320001244544983},{"id":"https://openalex.org/keywords/generative-model","display_name":"Generative model","score":0.41110000014305115},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.4034999907016754},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.40059998631477356}],"concepts":[{"id":"https://openalex.org/C101738243","wikidata":"https://www.wikidata.org/wiki/Q786435","display_name":"Autoencoder","level":3,"score":0.8880000114440918},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7294999957084656},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.6901999711990356},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.6365000009536743},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5769000053405762},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.46160000562667847},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.45320001244544983},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.41110000014305115},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.4034999907016754},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.40059998631477356},{"id":"https://openalex.org/C159877910","wikidata":"https://www.wikidata.org/wiki/Q2202883","display_name":"Autoregressive model","level":2,"score":0.3903000056743622},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.3813999891281128},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.37630000710487366},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3725999891757965},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.37139999866485596},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.31869998574256897},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.31029999256134033},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.27630001306533813},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.27320000529289246},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.2639999985694885},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.26030001044273376}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.14088","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.14088","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.14088","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.14088","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Video":[0,36],"generative":[1],"models":[2,111],"commonly":[3],"rely":[4],"on":[5,212],"latent":[6],"spaces":[7],"learned":[8],"by":[9,32],"3D":[10],"Variational":[11],"Autoencoders":[12],"(3D-VAEs).":[13],"However,":[14],"conventional":[15],"3D-VAEs":[16],"are":[17],"mainly":[18],"optimized":[19],"for":[20,103,109],"pixel-level":[21],"reconstruction,":[22],"which":[23],"can":[24,56],"limit":[25],"the":[26,124],"semantic":[27,129],"and":[28,44,62,89,106,131,147,159,163,201,207],"spatio-temporal":[29],"structure":[30],"captured":[31],"their":[33,53],"latents.":[34,204],"Meanwhile,":[35],"Foundation":[37],"Models":[38],"(VFMs)":[39],"such":[40],"as":[41,199],"V-JEPA":[42],"2":[43],"VideoMAEv2":[45],"show":[46,138],"strong":[47,142],"video":[48,64,86,203],"understanding":[49],"capabilities,":[50],"yet":[51],"whether":[52],"frozen":[54,85,125,196],"representations":[55,198],"be":[57,210],"transformed":[58],"into":[59],"compact,":[60],"reconstruction-capable,":[61],"generation-friendly":[63,202],"latents":[65,102],"remains":[66],"largely":[67],"unexplored.":[68],"We":[69],"answer":[70],"this":[71],"question":[72],"with":[73,92,123,161,184],"VideoRAE,":[74],"a":[75,84,93,118,177],"representation":[76,120],"autoencoder":[77,174],"that":[78,139],"leverages":[79],"multi-scale":[80],"hierarchical":[81],"features":[82],"from":[83],"foundation":[87],"encoder":[88],"compresses":[90],"them":[91],"lightweight":[94],"1D":[95],"self-attention":[96],"projector.":[97],"VideoRAE":[98,140,185],"supports":[99],"both":[100,145],"continuous":[101,146],"Diffusion":[104],"Transformers":[105],"discrete":[107,148],"tokens":[108],"autoregressive":[110],"via":[112],"multi-codebook":[113],"high-dimensional":[114],"quantization.":[115],"During":[116],"decoding,":[117],"local-and-global":[119],"alignment":[121],"objective":[122],"VFM":[126,197],"teacher":[127],"improves":[128],"preservation":[130],"enables":[132],"training":[133],"without":[134],"KL":[135],"regularization.":[136],"Experiments":[137],"achieves":[141],"reconstruction":[143],"in":[144],"regimes.":[149],"On":[150],"UCF-101,":[151],"it":[152],"obtains":[153],"state-of-the-art":[154],"class-to-video":[155],"gFVDs":[156],"of":[157],"40":[158],"93":[160],"AR":[162],"DiT":[164],"generators,":[165],"respectively,":[166],"while":[167],"converging":[168],"approximately":[169],"5x":[170],"faster":[171,188],"than":[172],"competing":[173],"baselines.":[175],"In":[176],"controlled":[178],"2B-scale":[179],"text-to-video":[180],"study,":[181],"replacing":[182],"LTX-VAE":[183],"leads":[186],"to":[187],"convergence":[189],"under":[190],"comparable":[191],"settings.":[192],"These":[193],"results":[194],"validate":[195],"versatile":[200],"The":[205],"model":[206],"code":[208],"will":[209],"released":[211],"https://zhxie0117.github.io/VideoRAE.":[213]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-17T00:00:00"}