{"id":"https://openalex.org/W4383109144","doi":"https://doi.org/10.1109/icra48891.2023.10161336","title":"Learning Video-Conditioned Policies for Unseen Manipulation Tasks","display_name":"Learning Video-Conditioned Policies for Unseen Manipulation Tasks","publication_year":2023,"publication_date":"2023-05-29","ids":{"openalex":"https://openalex.org/W4383109144","doi":"https://doi.org/10.1109/icra48891.2023.10161336"},"language":"en","primary_location":{"id":"doi:10.1109/icra48891.2023.10161336","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icra48891.2023.10161336","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://inria.hal.science/hal-04226856/document","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072932583","display_name":"Elliot Chane-Sane","orcid":null},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I2746051580","display_name":"Universit\u00e9 Paris Sciences et Lettres","ror":"https://ror.org/013cjyk83","country_code":"FR","type":"education","lineage":["https://openalex.org/I2746051580"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Elliot Chane-Sane","raw_affiliation_strings":["Inria, École normale supérieure, CNRS, PSL Research University,Paris,France,75005"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Inria, École normale supérieure, CNRS, PSL Research University,Paris,France,75005","institution_ids":["https://openalex.org/I2746051580","https://openalex.org/I1294671590"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109890544","display_name":"Cordelia Schmid","orcid":null},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I2746051580","display_name":"Universit\u00e9 Paris Sciences et Lettres","ror":"https://ror.org/013cjyk83","country_code":"FR","type":"education","lineage":["https://openalex.org/I2746051580"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Cordelia Schmid","raw_affiliation_strings":["Inria, École normale supérieure, CNRS, PSL Research University,Paris,France,75005"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Inria, École normale supérieure, CNRS, PSL Research University,Paris,France,75005","institution_ids":["https://openalex.org/I2746051580","https://openalex.org/I1294671590"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087781064","display_name":"Ivan Laptev","orcid":"https://orcid.org/0000-0001-7072-3325"},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I2746051580","display_name":"Universit\u00e9 Paris Sciences et Lettres","ror":"https://ror.org/013cjyk83","country_code":"FR","type":"education","lineage":["https://openalex.org/I2746051580"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Ivan Laptev","raw_affiliation_strings":["Inria, École normale supérieure, CNRS, PSL Research University,Paris,France,75005"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Inria, École normale supérieure, CNRS, PSL Research University,Paris,France,75005","institution_ids":["https://openalex.org/I2746051580","https://openalex.org/I1294671590"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":15,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"909","last_page":"916"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7696821093559265},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.7007458209991455},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6428523063659668},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6313259601593018},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.5541858673095703},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5005185604095459},{"id":"https://openalex.org/keywords/human\u2013robot-interaction","display_name":"Human\u2013robot interaction","score":0.4966748356819153},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.4943876266479492},{"id":"https://openalex.org/keywords/programming-by-demonstration","display_name":"Programming by demonstration","score":0.4610663056373596},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.4192761480808258},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.345122754573822},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07678702473640442}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7696821093559265},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.7007458209991455},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6428523063659668},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6313259601593018},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.5541858673095703},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5005185604095459},{"id":"https://openalex.org/C145460709","wikidata":"https://www.wikidata.org/wiki/Q859951","display_name":"Human\u2013robot interaction","level":3,"score":0.4966748356819153},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4943876266479492},{"id":"https://openalex.org/C2779038628","wikidata":"https://www.wikidata.org/wiki/Q7248497","display_name":"Programming by demonstration","level":3,"score":0.4610663056373596},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.4192761480808258},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.345122754573822},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07678702473640442},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/icra48891.2023.10161336","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icra48891.2023.10161336","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"},{"id":"pmh:oai:HAL:hal-04226856v1","is_oa":true,"landing_page_url":"https://inria.hal.science/hal-04226856","pdf_url":"https://inria.hal.science/hal-04226856/document","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2023 IEEE International Conference on Robotics and Automation (ICRA), May 2023, London, United Kingdom. pp.909-916, ⟨10.1109/ICRA48891.2023.10161336⟩","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"pmh:oai:HAL:hal-04226856v1","is_oa":true,"landing_page_url":"https://inria.hal.science/hal-04226856","pdf_url":"https://inria.hal.science/hal-04226856/document","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2023 IEEE International Conference on Robotics and Automation (ICRA), May 2023, London, United Kingdom. pp.909-916, ⟨10.1109/ICRA48891.2023.10161336⟩","raw_type":"info:eu-repo/semantics/conferenceObject"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.5099999904632568}],"awards":[{"id":"https://openalex.org/G4842108258","display_name":"Predicting future video content","funder_award_id":"ANR-21-FAI1-0002","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G5838787665","display_name":"PaRis Artificial Intelligence Research InstitutE","funder_award_id":"ANR-19-P3IA-0001","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G7124420487","display_name":null,"funder_award_id":"ANR19-P3IA-0001","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"}],"funders":[{"id":"https://openalex.org/F4320320883","display_name":"Agence Nationale de la Recherche","ror":"https://ror.org/00rbzpz17"},{"id":"https://openalex.org/F4320326256","display_name":"Grand \u00c9quipement National De Calcul Intensif","ror":"https://ror.org/0010d1q40"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4383109144.pdf"},"referenced_works_count":87,"referenced_works":["https://openalex.org/W2625366777","https://openalex.org/W2769112066","https://openalex.org/W2796136333","https://openalex.org/W2823112946","https://openalex.org/W2894766094","https://openalex.org/W2963435596","https://openalex.org/W2963703448","https://openalex.org/W2963802910","https://openalex.org/W2964036701","https://openalex.org/W2970377754","https://openalex.org/W2984008963","https://openalex.org/W2997876358","https://openalex.org/W3005680577","https://openalex.org/W3009295642","https://openalex.org/W3028830971","https://openalex.org/W3031840745","https://openalex.org/W3035524453","https://openalex.org/W3038298277","https://openalex.org/W3102502242","https://openalex.org/W3108330043","https://openalex.org/W3122520957","https://openalex.org/W3166396011","https://openalex.org/W3169291081","https://openalex.org/W3171176697","https://openalex.org/W3175995235","https://openalex.org/W3203511201","https://openalex.org/W3205423339","https://openalex.org/W3205786327","https://openalex.org/W3207832698","https://openalex.org/W4214717370","https://openalex.org/W4221159977","https://openalex.org/W4221160484","https://openalex.org/W4224330208","https://openalex.org/W4224912544","https://openalex.org/W4226167593","https://openalex.org/W4226435594","https://openalex.org/W4283218993","https://openalex.org/W4283788863","https://openalex.org/W4285218426","https://openalex.org/W4286858772","https://openalex.org/W4286989192","https://openalex.org/W4287126489","https://openalex.org/W4287599290","https://openalex.org/W4287778044","https://openalex.org/W4287812705","https://openalex.org/W4287994983","https://openalex.org/W4287998029","https://openalex.org/W4288020136","https://openalex.org/W4288333732","https://openalex.org/W4288350579","https://openalex.org/W4307225071","https://openalex.org/W4323640751","https://openalex.org/W6745420753","https://openalex.org/W6748012927","https://openalex.org/W6749916090","https://openalex.org/W6753060773","https://openalex.org/W6760439459","https://openalex.org/W6762137687","https://openalex.org/W6764438438","https://openalex.org/W6765008394","https://openalex.org/W6766848740","https://openalex.org/W6769035977","https://openalex.org/W6770887232","https://openalex.org/W6772034216","https://openalex.org/W6772111244","https://openalex.org/W6774314701","https://openalex.org/W6776700526","https://openalex.org/W6779463497","https://openalex.org/W6784212503","https://openalex.org/W6784862516","https://openalex.org/W6791353385","https://openalex.org/W6793841834","https://openalex.org/W6796289742","https://openalex.org/W6798184267","https://openalex.org/W6800774826","https://openalex.org/W6801810553","https://openalex.org/W6802366246","https://openalex.org/W6803941389","https://openalex.org/W6804244202","https://openalex.org/W6804404469","https://openalex.org/W6809850638","https://openalex.org/W6810080435","https://openalex.org/W6810361799","https://openalex.org/W6810640255","https://openalex.org/W6810655313","https://openalex.org/W6811554491","https://openalex.org/W6839390831"],"related_works":["https://openalex.org/W2081900870","https://openalex.org/W2371138613","https://openalex.org/W2048963458","https://openalex.org/W43109613","https://openalex.org/W3205513966","https://openalex.org/W2052971528","https://openalex.org/W3120459843","https://openalex.org/W4366547574","https://openalex.org/W3200191727","https://openalex.org/W4312312060"],"abstract_inverted_index":{"The":[0],"ability":[1],"to":[2,27,49,93,104,122,169,183],"specify":[3,28],"robot":[4,31,55,94,137,141,144,170,191,203,221],"commands":[5],"by":[6,34,153,193],"a":[7,19,35,38,61,82,113,197,216,237],"non-expert":[8],"user":[9],"is":[10,33,246],"critical":[11],"for":[12,157],"building":[13],"generalist":[14],"agents":[15],"capable":[16],"of":[17,22,37,89,115,218,227],"solving":[18],"large":[20],"variety":[21],"tasks.":[23],"One":[24],"convenient":[25],"way":[26],"the":[29,41,116,173,228],"intended":[30],"goal":[32],"video":[36,114,154,175],"person":[39],"demonstrating":[40],"target":[42,117],"task.":[43,118],"While":[44],"prior":[45],"work":[46],"typically":[47],"aims":[48],"imitate":[50],"human":[51,74,87,146,158,167,194,207],"demonstrations":[52,68,88,195],"performed":[53],"in":[54,70,148,172,196,236],"environments,":[56],"here":[57],"we":[58,100,125,164],"focus":[59],"on":[60,215],"more":[62],"realistic":[63],"and":[64,72,112,131,139,145,178,224],"challenging":[65,219,239],"setup":[66,241],"with":[67,206],"recorded":[69],"natural":[71],"diverse":[73],"environments.":[75],"We":[76,211],"propose":[77],"Video-conditioned":[78],"Policy":[79],"learning":[80],"(ViP),":[81],"data-driven":[83],"approach":[84,189,214],"that":[85],"maps":[86],"previously":[90],"unseen":[91],"tasks":[92,128],"manipulation":[95,222],"skills.":[96],"To":[97,119],"this":[98],"end,":[99],"learn":[101,132],"our":[102,133,149,185,188,213],"policy":[103,134],"generate":[105],"appropriate":[106],"actions":[107],"given":[108],"current":[109],"scene":[110],"observations":[111],"encourage":[120],"generalization":[121],"new":[123,238],"tasks,":[124],"avoid":[126],"particular":[127],"during":[129,209,248],"training":[130],"from":[135],"unlabelled":[136],"trajectories":[138,204],"corresponding":[140],"videos.":[142],"Both":[143],"videos":[147,168,171],"framework":[150],"are":[151],"represented":[152],"embeddings":[155,182],"pre-trained":[156],"action":[159],"recognition.":[160],"At":[161],"test":[162],"time":[163],"first":[165],"translate":[166],"common":[174],"embedding":[176],"space,":[177],"then":[179],"use":[180],"resulting":[181],"condition":[184],"policies.":[186],"Notably,":[187],"enables":[190],"control":[192],"zero-shot":[198,240],"manner,":[199],"i.e.,":[200],"without":[201],"using":[202],"paired":[205,244],"instructions":[208],"training.":[210,249],"validate":[212],"set":[217],"multi-task":[220],"environments":[223],"outperform":[225],"state":[226],"art.":[229],"Our":[230],"method":[231],"also":[232],"demonstrates":[233],"excellent":[234],"performance":[235],"where":[242],"no":[243],"data":[245],"used":[247]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":5}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}