{"id":"https://openalex.org/W7135125976","doi":"https://doi.org/10.48550/arxiv.2603.10448","title":"DiT4DiT: Jointly Modeling Video Dynamics and Actions for Generalizable Robot Control","display_name":"DiT4DiT: Jointly Modeling Video Dynamics and Actions for Generalizable Robot Control","publication_year":2026,"publication_date":"2026-03-11","ids":{"openalex":"https://openalex.org/W7135125976","doi":"https://doi.org/10.48550/arxiv.2603.10448"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.10448","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.10448","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.10448","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128881588","display_name":"Teli Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Teli","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128896676","display_name":"Jia Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Jia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128886891","display_name":"Zifan Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zifan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128918915","display_name":"Chuili Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Chunli","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128814710","display_name":"Andy Cui","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cui, Andy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128806441","display_name":"Junwei Liang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liang, Junwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5128870748","display_name":"Shuo Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Shuo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.22779999673366547,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.22779999673366547,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.20649999380111694,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.18150000274181366,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.5497000217437744},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.4702000021934509},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.4203999936580658},{"id":"https://openalex.org/keywords/robotics","display_name":"Robotics","score":0.3652999997138977},{"id":"https://openalex.org/keywords/segmentation","display_name":"Segmentation","score":0.358599990606308},{"id":"https://openalex.org/keywords/generative-model","display_name":"Generative model","score":0.33559998869895935},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.33390000462532043}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7301999926567078},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5547000169754028},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.5497000217437744},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4702000021934509},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.4203999936580658},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.3652999997138977},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.358599990606308},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.33559998869895935},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.33390000462532043},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.31529998779296875},{"id":"https://openalex.org/C113364801","wikidata":"https://www.wikidata.org/wiki/Q26674","display_name":"High fidelity","level":2,"score":0.2874000072479248},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.28700000047683716},{"id":"https://openalex.org/C202474056","wikidata":"https://www.wikidata.org/wiki/Q1931635","display_name":"Video tracking","level":3,"score":0.2824000120162964},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.28130000829696655},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2808000147342682},{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.27950000762939453},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.26829999685287476},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.26440000534057617},{"id":"https://openalex.org/C163294075","wikidata":"https://www.wikidata.org/wiki/Q581861","display_name":"Noise reduction","level":2,"score":0.25040000677108765}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.10448","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.10448","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.10448","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.10448","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Vision-Language-Action":[0],"(VLA)":[1],"models":[2,224],"have":[3],"emerged":[4],"as":[5,111,211],"a":[6,48,77,87,121],"promising":[7],"paradigm":[8],"for":[9,51,115,131,216],"robot":[10,217],"learning,":[11],"but":[12],"their":[13,55],"representations":[14],"are":[15,57],"still":[16],"largely":[17],"inherited":[18],"from":[19,29,103],"static":[20],"image-text":[21],"pretraining,":[22],"leaving":[23],"physical":[24],"dynamics":[25],"to":[26,203],"be":[27],"learned":[28],"comparatively":[30],"limited":[31],"action":[32,83,116,137],"data.":[33,173],"Generative":[34],"video":[35,78,105,132,207],"models,":[36],"by":[37,194,201],"contrast,":[38],"encode":[39],"rich":[40],"spatiotemporal":[41],"structure":[42],"and":[43,108,128,136,148,163,185,197,223],"implicit":[44],"physics,":[45],"making":[46],"them":[47,110],"compelling":[49],"foundation":[50],"robotic":[52],"manipulation.":[53],"But":[54],"potentials":[56],"not":[58],"fully":[59],"explored":[60],"in":[61,86],"the":[62,66,104,175],"literature.":[63],"To":[64],"bridge":[65],"gap,":[67],"we":[68],"introduce":[69],"DiT4DiT,":[70],"an":[71,82,212],"end-to-end":[72],"Video-Action":[73],"Model":[74],"that":[75,206],"couples":[76],"Diffusion":[79,84],"Transformer":[80,85],"with":[81,125],"unified":[88],"cascaded":[89],"framework.":[90],"Instead":[91],"of":[92,143,159],"relying":[93],"on":[94,161,165],"reconstructed":[95],"future":[96],"frames,":[97],"DiT4DiT":[98,151,190],"extracts":[99],"intermediate":[100],"denoising":[101],"features":[102],"generation":[106,208],"process":[107],"uses":[109],"temporally":[112],"grounded":[113],"conditions":[114],"prediction.":[117],"We":[118,220],"further":[119],"propose":[120],"dual":[122],"flow-matching":[123],"objective":[124],"decoupled":[126],"timesteps":[127],"noise":[129],"scales":[130],"prediction,":[133],"hidden-state":[134],"extraction,":[135],"inference,":[138],"enabling":[139],"coherent":[140],"joint":[141],"training":[142,172],"both":[144],"modules.":[145],"Across":[146],"simulation":[147],"real-world":[149,183],"benchmarks,":[150],"achieves":[152],"state-of-the-art":[153],"results,":[154],"reaching":[155],"average":[156],"success":[157],"rates":[158],"98.6%":[160],"LIBERO":[162],"50.8%":[164],"RoboCasa":[166],"GR1":[167],"while":[168],"using":[169],"substantially":[170],"less":[171],"On":[174],"Unitree":[176],"G1":[177],"robot,":[178],"it":[179],"also":[180],"delivers":[181],"superior":[182],"performance":[184],"strong":[186],"zero-shot":[187],"generalization.":[188],"Importantly,":[189],"improves":[191],"sample":[192],"efficiency":[193],"over":[195],"10x":[196],"speeds":[198],"up":[199,202],"convergence":[200],"7x,":[204],"demonstrating":[205],"can":[209],"serve":[210],"effective":[213],"scaling":[214],"proxy":[215],"policy":[218],"learning.":[219],"release":[221],"code":[222],"at":[225],"https://dit4dit.github.io/.":[226]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-13T00:00:00"}