{"id":"https://openalex.org/W7214924518","doi":"https://doi.org/10.48550/arxiv.2609.36870","title":"VidAct: Learning Manipulation from In-the-Wild Videos with Object-Centric 3D Awareness","display_name":"VidAct: Learning Manipulation from In-the-Wild Videos with Object-Centric 3D Awareness","publication_year":2026,"publication_date":"2026-09-29","ids":{"openalex":"https://openalex.org/W7214924518","doi":"https://doi.org/10.48550/arxiv.2609.36870"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.36870","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.36870","pdf_url":"https://arxiv.org/pdf/2609.36870","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.36870","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5153569890","display_name":"Hang Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Li, Hang","raw_affiliation_strings":["Technical University of Munich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technical University of Munich","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101379426","display_name":"Mingxin Zhang","orcid":"https://orcid.org/0000-0003-4877-047X","observed_orcids":["https://orcid.org/0000-0003-4877-047X"]},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Zhang, Mingxin","raw_affiliation_strings":["Technical University of Munich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technical University of Munich","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153607688","display_name":"Zihan Wu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Wu, Zihan","raw_affiliation_strings":["Technical University of Munich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technical University of Munich","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153614494","display_name":"Yang Tian","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tian, Yang","raw_affiliation_strings":["Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153620046","display_name":"Dong Chen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Dong","raw_affiliation_strings":["Huawei Heisenberg Research Center"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Heisenberg Research Center","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052009653","display_name":"Fengyi Shen","orcid":"https://orcid.org/0000-0001-7621-9779","observed_orcids":["https://orcid.org/0000-0001-7621-9779","https://orcid.org/0000-0001-8391-4104"]},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Shen, Fengyi","raw_affiliation_strings":["Huawei Heisenberg Research Center","Technical University of Munich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Heisenberg Research Center","institution_ids":[]},{"raw_affiliation_string":"Technical University of Munich","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153595751","display_name":"Yuan Meng","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Meng, Yuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073985865","display_name":"Xiangtong Yao","orcid":"https://orcid.org/0000-0003-2556-3072","observed_orcids":["https://orcid.org/0000-0003-2556-3072"]},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Yao, Xiangtong","raw_affiliation_strings":["Technical University of Munich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technical University of Munich","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153554368","display_name":"Heng Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhang, Heng","raw_affiliation_strings":["Huawei CloudRobo Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei CloudRobo Lab","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153624409","display_name":"Ziyuan Liu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Ziyuan","raw_affiliation_strings":["Huawei Heisenberg Research Center"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Heisenberg Research Center","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060444894","display_name":"Zhenshan Bing","orcid":"https://orcid.org/0000-0002-0896-2517","observed_orcids":["https://orcid.org/0000-0002-0896-2517"]},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]},{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN","DE"],"is_corresponding":false,"raw_author_name":"Bing, Zhenshan","raw_affiliation_strings":["Nanjing University Real World","Technical University of Munich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanjing University Real World","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"Technical University of Munich","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5153622169","display_name":"Alois Knoll","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Knoll, Alois","raw_affiliation_strings":["Technical University of Munich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technical University of Munich","institution_ids":["https://openalex.org/I62916508"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.975600004196167,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.975600004196167,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.02319999970495701,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10719","display_name":"3D Shape Modeling and Analysis","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1704","display_name":"Computer Graphics and Computer-Aided Design"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/video-demonstration","display_name":"video demonstration","score":0.6940000057220459},{"id":"https://openalex.org/keywords/monocular-video","display_name":"monocular video","score":0.6019999980926514},{"id":"https://openalex.org/keywords/motion-reconstruction","display_name":"motion reconstruction","score":0.527999997138977},{"id":"https://openalex.org/keywords/sim-to-real-transfer","display_name":"sim-to-real transfer","score":0.7360000014305115},{"id":"https://openalex.org/keywords/zero-shot-deployment","display_name":"zero-shot deployment","score":0.843999981880188},{"id":"https://openalex.org/keywords/object-grounding","display_name":"object grounding","score":0.6650000214576721},{"id":"https://openalex.org/keywords/in-the-wild-videos","display_name":"in-the-wild videos","score":0.5879999995231628}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8034999966621399},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.7612000107765198},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7459999918937683},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.7164999842643738},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.6601999998092651},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5177000164985657},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5080000162124634},{"id":"https://openalex.org/C104114177","wikidata":"https://www.wikidata.org/wiki/Q79782","display_name":"Motion (physics)","level":2,"score":0.5008999705314636},{"id":"https://openalex.org/C2780575108","wikidata":"https://www.wikidata.org/wiki/Q7316652","display_name":"Retargeting","level":2,"score":0.49050000309944153},{"id":"https://openalex.org/C202474056","wikidata":"https://www.wikidata.org/wiki/Q1931635","display_name":"Video tracking","level":3,"score":0.42879998683929443},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.4099000096321106},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.3865000009536743},{"id":"https://openalex.org/C2780139006","wikidata":"https://www.wikidata.org/wiki/Q1493902","display_name":"Key frame","level":3,"score":0.3846000134944916},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.35260000824928284},{"id":"https://openalex.org/C131979681","wikidata":"https://www.wikidata.org/wiki/Q1899648","display_name":"Point cloud","level":2,"score":0.3434000015258789},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.33959999680519104},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.3199999928474426},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.29660001397132874},{"id":"https://openalex.org/C146159030","wikidata":"https://www.wikidata.org/wiki/Q7625099","display_name":"Structure from motion","level":3,"score":0.29440000653266907},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.29319998621940613},{"id":"https://openalex.org/C52102323","wikidata":"https://www.wikidata.org/wiki/Q1671968","display_name":"Pose","level":2,"score":0.2743000090122223},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.27410000562667847},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.26100000739097595},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.2587999999523163},{"id":"https://openalex.org/C65909025","wikidata":"https://www.wikidata.org/wiki/Q1945033","display_name":"Monocular","level":2,"score":0.25870001316070557},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2556000053882599},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.25029999017715454}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.36870","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.36870","pdf_url":"https://arxiv.org/pdf/2609.36870","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.36870","doi":"https://doi.org/10.48550/arxiv.2609.36870","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.36870","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.36870","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.36870","pdf_url":"https://arxiv.org/pdf/2609.36870","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214924518.pdf","grobid_xml":"https://content.openalex.org/works/W7214924518.grobid-xml"},"referenced_works_count":10,"referenced_works":["https://openalex.org/W4414674749","https://openalex.org/W4414975320","https://openalex.org/W4415313797","https://openalex.org/W4416551737","https://openalex.org/W7092289344","https://openalex.org/W7117736774","https://openalex.org/W7128604160","https://openalex.org/W7128614621","https://openalex.org/W7130590664","https://openalex.org/W7140999116"],"related_works":["https://openalex.org/W4413144616","https://openalex.org/W4416075297","https://openalex.org/W4415274204","https://openalex.org/W4387929106","https://openalex.org/W7138091585","https://openalex.org/W7128487011","https://openalex.org/W3102502242","https://openalex.org/W4417032950","https://openalex.org/W4403888324","https://openalex.org/W7213563823"],"abstract_inverted_index":{"Video":[0],"demonstrations":[1],"offer":[2],"a":[3,87],"scalable":[4],"alternative":[5],"to":[6,32,34,143],"costly":[7],"robot":[8],"data":[9],"for":[10,66,138],"learning":[11],"manipulation,":[12],"yet":[13],"existing":[14],"reconstruction-based":[15],"approaches":[16],"often":[17,51],"rely":[18],"on":[19,188],"constrained":[20],"camera":[21,130],"viewpoints":[22],"or":[23],"human-to-robot":[24],"retargeting,":[25],"while":[26,147,172,212],"the":[27,40,48,117,120,140,154],"reconstructed":[28,141],"trajectories":[29],"are":[30],"difficult":[31],"adapt":[33],"new":[35],"objects":[36],"configurations":[37,146],"without":[38],"distorting":[39],"trajectory":[41,136,214,218],"shape.":[42,151],"Another":[43],"key":[44,102,155],"limitation":[45],"is":[46],"that":[47,80],"resulting":[49],"policies":[50,85],"lack":[52],"precise":[53,67],"object-level":[54],"3D":[55,185,204],"geometry":[56],"awareness,":[57],"limiting":[58],"object":[59,62,107,122,145,182],"grounding":[60],"and":[61,93,109,115,127,184,192,199,209],"shape":[63,222],"awareness":[64],"critical":[65],"manipulation.":[68],"To":[69],"bridge":[70],"these":[71],"gaps,":[72],"we":[73],"propose":[74],"VidAct,":[75],"an":[76,169],"efficient":[77],"video-to-robot":[78],"framework":[79],"learns":[81],"object-centric,":[82],"3D-aware":[83],"manipulation":[84],"from":[86,111],"single":[88],"monocular":[89],"video":[90,197],"per":[91],"task":[92,171],"enables":[94,216],"zero-shot":[95,200],"real-world":[96],"deployment.":[97,201],"VidAct":[98,105,133,158],"consists":[99],"of":[100],"three":[101],"components.":[103],"First,":[104],"reconstructs":[106],"meshes":[108],"motion":[110,118,142,150],"arbitrary":[112],"demo":[113],"videos":[114,194],"canonicalizes":[116],"in":[119],"static":[121],"frame,":[123],"avoiding":[124],"embodiment-specific":[125],"retargeting":[126],"accommodating":[128],"diverse":[129],"viewpoints.":[131],"Second,":[132],"employ":[134],"residual":[135,213],"transfer":[137,215],"adapting":[139],"novel":[144],"preserving":[148],"its":[149],"Finally,":[152],"as":[153,168],"policy-learning":[156],"component,":[157],"predicts":[159],"simulation-provided":[160],"privileged":[161],"complete-object":[162,203],"point":[163],"clouds":[164],"at":[165],"each":[166],"frame":[167],"auxiliary":[170],"retaining":[173],"RGB-only":[174],"deployment,":[175],"providing":[176],"dense":[177],"object-centric":[178],"supervision":[179,205],"over":[180],"both":[181],"pose":[183],"geometry.":[186],"Experiments":[187],"human,":[189],"robot,":[190],"generated,":[191],"internet":[193],"demonstrate":[195],"broad":[196],"applicability":[198],"Per-frame":[202],"improves":[206],"policy":[207],"generalization":[208],"sim-to-real":[210],"success,":[211],"reliable":[217],"adaptation":[219],"with":[220],"better":[221],"preservation.":[223]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-01T00:00:00"}