{"id":"https://openalex.org/W4403575288","doi":"https://doi.org/10.48550/arxiv.2410.11792","title":"OKAMI: Teaching Humanoid Robots Manipulation Skills through Single Video Imitation","display_name":"OKAMI: Teaching Humanoid Robots Manipulation Skills through Single Video Imitation","publication_year":2024,"publication_date":"2024-10-15","ids":{"openalex":"https://openalex.org/W4403575288","doi":"https://doi.org/10.48550/arxiv.2410.11792"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.11792","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.11792","pdf_url":"https://arxiv.org/pdf/2410.11792","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.11792","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061176165","display_name":"Jinhan Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li, Jinhan","raw_affiliation_strings":["NVIDIA Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA Research","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101638642","display_name":"Yifeng Zhu","orcid":"https://orcid.org/0000-0003-2580-5748","observed_orcids":["https://orcid.org/0000-0003-2580-5748"]},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhu, Yifeng","raw_affiliation_strings":["NVIDIA Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA Research","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102516944","display_name":"Yuqi Xie","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xie, Yuqi","raw_affiliation_strings":["NVIDIA Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA Research","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060626901","display_name":"Zhenyu Jiang","orcid":"https://orcid.org/0000-0001-8497-3405","observed_orcids":["https://orcid.org/0000-0001-8497-3405","https://orcid.org/0009-0002-5531-3971"]},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiang, Zhenyu","raw_affiliation_strings":["NVIDIA Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA Research","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075135654","display_name":"Mingyo Seo","orcid":"https://orcid.org/0000-0003-0966-0462","observed_orcids":["https://orcid.org/0000-0003-0966-0462"]},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Seo, Mingyo","raw_affiliation_strings":["NVIDIA Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA Research","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052269532","display_name":"Georgios Pavlakos","orcid":"https://orcid.org/0000-0001-5821-1909","observed_orcids":["https://orcid.org/0000-0001-5821-1909"]},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pavlakos, Georgios","raw_affiliation_strings":["NVIDIA Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA Research","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5030826237","display_name":"Yuke Zhu","orcid":"https://orcid.org/0000-0002-9198-2227","observed_orcids":["https://orcid.org/0000-0002-9198-2227"]},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhu, Yuke","raw_affiliation_strings":["NVIDIA Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA Research","institution_ids":["https://openalex.org/I4210127875"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210127875"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":38,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9692000150680542,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9692000150680542,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.02449999935925007,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.0010000000474974513,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/humanoid-robots","display_name":"humanoid robots","score":0.8930000066757202},{"id":"https://openalex.org/keywords/manipulation-skills","display_name":"manipulative skills","score":0.8259999752044678},{"id":"https://openalex.org/keywords/rgb-d-video","display_name":"RGB-D video","score":0.6370000243186951},{"id":"https://openalex.org/keywords/visuomotor-policies","display_name":"visuomotor policies","score":0.8679999709129333},{"id":"https://openalex.org/keywords/human-motion-imitation","display_name":"human motion imitation","score":0.3970000147819519}],"concepts":[{"id":"https://openalex.org/C126388530","wikidata":"https://www.wikidata.org/wiki/Q1131737","display_name":"Imitation","level":2,"score":0.9017671346664429},{"id":"https://openalex.org/C60692881","wikidata":"https://www.wikidata.org/wiki/Q584529","display_name":"Humanoid robot","level":3,"score":0.6923381686210632},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5488517880439758},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.503323495388031},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.5002405643463135},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3641173243522644},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3531140089035034},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35253584384918213},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.07233232259750366}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.11792","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.11792","pdf_url":"https://arxiv.org/pdf/2410.11792","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.11792","doi":"https://doi.org/10.48550/arxiv.2410.11792","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.11792","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.11792","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.11792","pdf_url":"https://arxiv.org/pdf/2410.11792","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403575288.pdf","grobid_xml":"https://content.openalex.org/works/W4403575288.grobid-xml"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W2786036844","https://openalex.org/W2906540835","https://openalex.org/W2942930375","https://openalex.org/W3039055967","https://openalex.org/W3088304681","https://openalex.org/W3187722890","https://openalex.org/W4307078757","https://openalex.org/W4322616306","https://openalex.org/W4323066621","https://openalex.org/W4324128075","https://openalex.org/W4383108575","https://openalex.org/W4385430674","https://openalex.org/W4387225729","https://openalex.org/W4390306236","https://openalex.org/W4399270642","https://openalex.org/W4399794218","https://openalex.org/W4400342723","https://openalex.org/W4401415732","https://openalex.org/W4402354017","https://openalex.org/W4402354045","https://openalex.org/W4402354112","https://openalex.org/W4403906568","https://openalex.org/W4405787755","https://openalex.org/W4413925191","https://openalex.org/W4413925671"],"related_works":["https://openalex.org/W4399270642","https://openalex.org/W4417032950","https://openalex.org/W4413925044","https://openalex.org/W7131372932","https://openalex.org/W4384264703","https://openalex.org/W4414990022","https://openalex.org/W3122520957","https://openalex.org/W7222727425","https://openalex.org/W4414991924","https://openalex.org/W3163183849"],"abstract_inverted_index":{"We":[0,16],"study":[1],"the":[2,38,48,53,79,101,129],"problem":[3],"of":[4,40,126],"teaching":[5],"humanoid":[6,49],"robots":[7],"manipulation":[8,24],"skills":[9],"by":[10],"imitating":[11],"from":[12,26,107],"single":[13,28],"video":[14,30,59],"demonstrations.":[15],"introduce":[17],"OKAMI,":[18],"a":[19,23,27,33],"method":[20],"that":[21,89],"generates":[22],"plan":[25],"RGB-D":[29,58],"and":[31,77,82,97],"derives":[32],"policy":[34],"for":[35,131],"execution.":[36],"At":[37],"heart":[39],"our":[41,140],"approach":[42],"is":[43],"object-aware":[44],"retargeting,":[45],"which":[46,120],"enables":[47],"robot":[50],"to":[51,62,73,115],"mimic":[52],"human":[54],"motions":[55,81],"in":[56],"an":[57,122],"while":[60],"adjusting":[61],"different":[63],"object":[64],"locations":[65],"during":[66],"deployment.":[67],"OKAMI":[68,90,110],"uses":[69],"open-world":[70,105],"vision":[71],"models":[72],"identify":[74],"task-relevant":[75],"objects":[76],"retarget":[78],"body":[80],"hand":[83],"poses":[84],"separately.":[85],"Our":[86],"experiments":[87],"show":[88],"achieves":[91],"strong":[92],"generalizations":[93],"across":[94],"varying":[95],"visual":[96],"spatial":[98],"conditions,":[99],"outperforming":[100],"state-of-the-art":[102],"baseline":[103],"on":[104,139],"imitation":[106],"observation.":[108],"Furthermore,":[109],"rollout":[111],"trajectories":[112],"are":[113],"leveraged":[114],"train":[116],"closed-loop":[117],"visuomotor":[118],"policies,":[119],"achieve":[121],"average":[123],"success":[124],"rate":[125],"79.2%":[127],"without":[128],"need":[130],"labor-intensive":[132],"teleoperation.":[133],"More":[134],"videos":[135],"can":[136],"be":[137],"found":[138],"website":[141],"https://ut-austin-rpl.github.io/OKAMI/.":[142]},"counts_by_year":[{"year":2026,"cited_by_count":24},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-10-20T00:00:00"}