{"id":"https://openalex.org/W3122520957","doi":"https://doi.org/10.1109/iros51168.2021.9636080","title":"Learning by Watching: Physical Imitation of Manipulation Skills from Human Videos","display_name":"Learning by Watching: Physical Imitation of Manipulation Skills from Human Videos","publication_year":2021,"publication_date":"2021-09-27","ids":{"openalex":"https://openalex.org/W3122520957","doi":"https://doi.org/10.1109/iros51168.2021.9636080","mag":"3122520957"},"language":"en","primary_location":{"id":"doi:10.1109/iros51168.2021.9636080","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iros51168.2021.9636080","pdf_url":null,"source":{"id":"https://openalex.org/S4363607734","display_name":"2021 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5104123870","display_name":"Haoyu Xiong","orcid":"https://orcid.org/0009-0002-5972-9405"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]},{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA","CN"],"is_corresponding":false,"raw_author_name":"Haoyu Xiong","raw_affiliation_strings":["Tianjin University","University of Toronto & Vector Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tianjin University","institution_ids":["https://openalex.org/I162868743"]},{"raw_affiliation_string":"University of Toronto & Vector Institute","institution_ids":["https://openalex.org/I185261750","https://openalex.org/I4210127509"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053537085","display_name":"Quanzhou Li","orcid":null},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Quanzhou Li","raw_affiliation_strings":["University of Toronto & Vector Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto & Vector Institute","institution_ids":["https://openalex.org/I185261750","https://openalex.org/I4210127509"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074633339","display_name":"Yun-Chun Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Yun-Chun Chen","raw_affiliation_strings":["University of Toronto & Vector Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto & Vector Institute","institution_ids":["https://openalex.org/I185261750","https://openalex.org/I4210127509"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064794691","display_name":"Homanga Bharadhwaj","orcid":"https://orcid.org/0000-0001-5056-8516"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Homanga Bharadhwaj","raw_affiliation_strings":["University of Toronto & Vector Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto & Vector Institute","institution_ids":["https://openalex.org/I185261750","https://openalex.org/I4210127509"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101182654","display_name":"Samarth Sinha","orcid":null},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Samarth Sinha","raw_affiliation_strings":["University of Toronto & Vector Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto & Vector Institute","institution_ids":["https://openalex.org/I185261750","https://openalex.org/I4210127509"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061193324","display_name":"Animesh Garg","orcid":"https://orcid.org/0000-0003-0482-4296"},"institutions":[{"id":"https://openalex.org/I1304085615","display_name":"Nvidia (United Kingdom)","ror":"https://ror.org/02kr42612","country_code":"GB","type":"company","lineage":["https://openalex.org/I1304085615","https://openalex.org/I4210127875"]},{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA","GB"],"is_corresponding":false,"raw_author_name":"Animesh Garg","raw_affiliation_strings":["Nvidia","University of Toronto & Vector Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nvidia","institution_ids":["https://openalex.org/I1304085615"]},{"raw_affiliation_string":"University of Toronto & Vector Institute","institution_ids":["https://openalex.org/I185261750","https://openalex.org/I4210127509"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":44,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"7827","last_page":"7834"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8096113204956055},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6737056970596313},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6354058384895325},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.6312389373779297},{"id":"https://openalex.org/keywords/imitation","display_name":"Imitation","score":0.629713237285614},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6271052956581116},{"id":"https://openalex.org/keywords/salient","display_name":"Salient","score":0.5666790008544922},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.523127019405365},{"id":"https://openalex.org/keywords/unsupervised-learning","display_name":"Unsupervised learning","score":0.4757368564605713},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4434780180454254},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.43287935853004456},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3965897560119629}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8096113204956055},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6737056970596313},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6354058384895325},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.6312389373779297},{"id":"https://openalex.org/C126388530","wikidata":"https://www.wikidata.org/wiki/Q1131737","display_name":"Imitation","level":2,"score":0.629713237285614},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6271052956581116},{"id":"https://openalex.org/C2780719617","wikidata":"https://www.wikidata.org/wiki/Q1030752","display_name":"Salient","level":2,"score":0.5666790008544922},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.523127019405365},{"id":"https://openalex.org/C8038995","wikidata":"https://www.wikidata.org/wiki/Q1152135","display_name":"Unsupervised learning","level":2,"score":0.4757368564605713},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4434780180454254},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43287935853004456},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3965897560119629},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/iros51168.2021.9636080","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iros51168.2021.9636080","pdf_url":null,"source":{"id":"https://openalex.org/S4363607734","display_name":"2021 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":99,"referenced_works":["https://openalex.org/W1812075590","https://openalex.org/W2081034428","https://openalex.org/W2160609165","https://openalex.org/W2583137229","https://openalex.org/W2768959015","https://openalex.org/W2769112066","https://openalex.org/W2781726626","https://openalex.org/W2787666871","https://openalex.org/W2806130867","https://openalex.org/W2807725536","https://openalex.org/W2808849637","https://openalex.org/W2887997593","https://openalex.org/W2889347284","https://openalex.org/W2890967717","https://openalex.org/W2894766094","https://openalex.org/W2900152462","https://openalex.org/W2908470496","https://openalex.org/W2913937523","https://openalex.org/W2949678110","https://openalex.org/W2950111065","https://openalex.org/W2955368974","https://openalex.org/W2960029743","https://openalex.org/W2962787969","https://openalex.org/W2962793481","https://openalex.org/W2962936820","https://openalex.org/W2962947361","https://openalex.org/W2962981304","https://openalex.org/W2963022858","https://openalex.org/W2963330667","https://openalex.org/W2963390419","https://openalex.org/W2963419579","https://openalex.org/W2963516265","https://openalex.org/W2963634205","https://openalex.org/W2963669336","https://openalex.org/W2963703448","https://openalex.org/W2963767194","https://openalex.org/W2963802910","https://openalex.org/W2963890275","https://openalex.org/W2963913081","https://openalex.org/W2964001908","https://openalex.org/W2964021598","https://openalex.org/W2970415880","https://openalex.org/W2970705309","https://openalex.org/W2971202257","https://openalex.org/W2977481643","https://openalex.org/W2981344907","https://openalex.org/W2981479880","https://openalex.org/W2985060393","https://openalex.org/W3010709437","https://openalex.org/W3030035469","https://openalex.org/W3032377877","https://openalex.org/W3035756007","https://openalex.org/W3038245394","https://openalex.org/W3038298277","https://openalex.org/W3039055967","https://openalex.org/W3086412568","https://openalex.org/W3090369311","https://openalex.org/W3101283005","https://openalex.org/W3102502242","https://openalex.org/W3110682054","https://openalex.org/W3121989526","https://openalex.org/W3147742130","https://openalex.org/W3159890710","https://openalex.org/W3208212324","https://openalex.org/W3210246279","https://openalex.org/W4287553698","https://openalex.org/W4287599290","https://openalex.org/W4288020136","https://openalex.org/W4288294128","https://openalex.org/W4288322145","https://openalex.org/W4289294484","https://openalex.org/W4300799055","https://openalex.org/W6638315535","https://openalex.org/W6732465633","https://openalex.org/W6734074887","https://openalex.org/W6740801417","https://openalex.org/W6745992979","https://openalex.org/W6747473740","https://openalex.org/W6750298367","https://openalex.org/W6752338937","https://openalex.org/W6752601656","https://openalex.org/W6754162236","https://openalex.org/W6754539604","https://openalex.org/W6754988648","https://openalex.org/W6755243447","https://openalex.org/W6756256016","https://openalex.org/W6764529735","https://openalex.org/W6765399202","https://openalex.org/W6765407481","https://openalex.org/W6765456200","https://openalex.org/W6766848740","https://openalex.org/W6768511085","https://openalex.org/W6769596995","https://openalex.org/W6783068448","https://openalex.org/W6784862516","https://openalex.org/W6785753252","https://openalex.org/W6786816963","https://openalex.org/W6786884049","https://openalex.org/W6795104053"],"related_works":["https://openalex.org/W3123344745","https://openalex.org/W3022038857","https://openalex.org/W4221136938","https://openalex.org/W4211044943","https://openalex.org/W4302303815","https://openalex.org/W3208584567","https://openalex.org/W2908875379","https://openalex.org/W3044458868","https://openalex.org/W3161440429","https://openalex.org/W3192794374"],"abstract_inverted_index":{"Learning":[0,36],"from":[1,48],"visual":[2],"data":[3],"opens":[4],"the":[5,53,65,71,87,94,113,142,172],"potential":[6],"to":[7,82,85,92],"accrue":[8],"a":[9,49,121],"large":[10],"range":[11],"of":[12,22,58,144],"manipulation":[13,151],"behaviors":[14],"by":[15,37],"leveraging":[16],"human":[17,66,81],"demonstrations":[18],"without":[19],"specifying":[20,52],"each":[21],"them":[23],"mathe-matically,":[24],"but":[25],"rather":[26],"through":[27,46],"natural":[28],"task":[29],"specification.":[30],"In":[31],"this":[32],"paper,":[33],"we":[34],"present":[35],"Watching":[38],"(LbW),":[39],"an":[40],"algorithmic":[41],"framework":[42,78,147],"for":[43,102,134],"policy":[44,138],"learning":[45,103],"imitation":[47],"single":[50],"video":[51],"task.":[54],"The":[55,117],"key":[56],"insights":[57],"our":[59,77,106,145,167],"method":[60,168],"are":[61,100,179],"two-fold.":[62],"First,":[63],"since":[64],"arms":[67],"may":[68],"not":[69],"have":[70],"same":[72],"morphology":[73,88],"as":[74],"robot":[75,83,115,150],"arms,":[76],"learns":[79],"unsupervised":[80,109],"translation":[84],"overcome":[86],"mis-match":[89],"issue.":[90],"Second,":[91],"capture":[93],"details":[95],"in":[96],"salient":[97],"regions":[98],"that":[99,124,166],"crucial":[101],"state":[104],"representations,":[105],"model":[107],"performs":[108,169],"keypoint":[110],"detection":[111],"on":[112,148],"translated":[114],"videos.":[116],"detected":[118],"keypoints":[119],"form":[120],"structured":[122],"representation":[123],"contains":[125],"semantically":[126],"meaningful":[127],"information":[128],"and":[129,137,159,177],"can":[130],"be":[131],"used":[132],"directly":[133],"computing":[135],"reward":[136],"learning.":[139],"We":[140],"evaluate":[141],"effectiveness":[143],"LbW":[146],"five":[149],"tasks,":[152],"including":[153],"reaching,":[154],"pushing,":[155],"sliding,":[156],"coffee":[157],"making,":[158],"drawer":[160],"closing.":[161],"Extensive":[162],"experimental":[163],"evaluations":[164],"demonstrate":[165],"favorably":[170],"against":[171],"state-of-the-art":[173],"approaches.":[174],"More":[175],"results":[176],"analysis":[178],"available":[180],"at":[181],"pair.toronto.edu/lbw-kp/.":[182]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":21},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":1}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}