{"id":"https://openalex.org/W4403277141","doi":"https://doi.org/10.1109/lra.2024.3477090","title":"GPT-4V(ision) for Robotics: Multimodal Task Planning From Human Demonstration","display_name":"GPT-4V(ision) for Robotics: Multimodal Task Planning From Human Demonstration","publication_year":2024,"publication_date":"2024-10-09","ids":{"openalex":"https://openalex.org/W4403277141","doi":"https://doi.org/10.1109/lra.2024.3477090"},"language":"en","primary_location":{"id":"doi:10.1109/lra.2024.3477090","is_oa":true,"landing_page_url":"https://doi.org/10.1109/lra.2024.3477090","pdf_url":null,"source":{"id":"https://openalex.org/S4210169774","display_name":"IEEE Robotics and Automation Letters","issn_l":"2377-3766","issn":["2377-3766","2377-3774"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics and Automation Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1109/lra.2024.3477090","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5087791577","display_name":"Naoki Wake","orcid":"https://orcid.org/0000-0001-8278-2373"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210116723","display_name":"Robotics Research (United States)","ror":"https://ror.org/020w2fr77","country_code":"US","type":"company","lineage":["https://openalex.org/I4210116723"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Naoki Wake","raw_affiliation_strings":["Applied Robotics Research, Microsoft, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0000-0001-8278-2373","affiliations":[{"raw_affiliation_string":"Applied Robotics Research, Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210116723"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011238975","display_name":"Atsushi Kanehira","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210116723","display_name":"Robotics Research (United States)","ror":"https://ror.org/020w2fr77","country_code":"US","type":"company","lineage":["https://openalex.org/I4210116723"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Atsushi Kanehira","raw_affiliation_strings":["Applied Robotics Research, Microsoft, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0005-0157-7500","affiliations":[{"raw_affiliation_string":"Applied Robotics Research, Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210116723"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035650290","display_name":"Kazuhiro Sasabuchi","orcid":"https://orcid.org/0000-0002-5408-3089"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210116723","display_name":"Robotics Research (United States)","ror":"https://ror.org/020w2fr77","country_code":"US","type":"company","lineage":["https://openalex.org/I4210116723"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kazuhiro Sasabuchi","raw_affiliation_strings":["Applied Robotics Research, Microsoft, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-5408-3089","affiliations":[{"raw_affiliation_string":"Applied Robotics Research, Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210116723"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047085140","display_name":"Jun Takamatsu","orcid":"https://orcid.org/0000-0001-7457-2878"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210116723","display_name":"Robotics Research (United States)","ror":"https://ror.org/020w2fr77","country_code":"US","type":"company","lineage":["https://openalex.org/I4210116723"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jun Takamatsu","raw_affiliation_strings":["Applied Robotics Research, Microsoft, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0000-0001-7457-2878","affiliations":[{"raw_affiliation_string":"Applied Robotics Research, Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210116723"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016258462","display_name":"Katsushi Ikeuchi","orcid":"https://orcid.org/0000-0001-9758-9357"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210116723","display_name":"Robotics Research (United States)","ror":"https://ror.org/020w2fr77","country_code":"US","type":"company","lineage":["https://openalex.org/I4210116723"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Katsushi Ikeuchi","raw_affiliation_strings":["Applied Robotics Research, Microsoft, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0000-0001-9758-9357","affiliations":[{"raw_affiliation_string":"Applied Robotics Research, Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210116723"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":11.8968,"has_fulltext":false,"cited_by_count":64,"citation_normalized_percentile":{"value":0.99420877,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"9","issue":"11","first_page":"10567","last_page":"10574"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.8288999795913696,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.8288999795913696,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.8098000288009644,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robotics","display_name":"Robotics","score":0.7312030792236328},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.661120593547821},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6384459137916565},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5236296653747559},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.4994330406188965},{"id":"https://openalex.org/keywords/multimodal-therapy","display_name":"Multimodal therapy","score":0.45457762479782104},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.2644472122192383},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.24995484948158264},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.2425546646118164},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.16140413284301758},{"id":"https://openalex.org/keywords/psychotherapist","display_name":"Psychotherapist","score":0.05411013960838318}],"concepts":[{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.7312030792236328},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.661120593547821},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6384459137916565},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5236296653747559},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4994330406188965},{"id":"https://openalex.org/C4441509","wikidata":"https://www.wikidata.org/wiki/Q6418787","display_name":"Multimodal therapy","level":2,"score":0.45457762479782104},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.2644472122192383},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.24995484948158264},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2425546646118164},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.16140413284301758},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.05411013960838318}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/lra.2024.3477090","is_oa":true,"landing_page_url":"https://doi.org/10.1109/lra.2024.3477090","pdf_url":null,"source":{"id":"https://openalex.org/S4210169774","display_name":"IEEE Robotics and Automation Letters","issn_l":"2377-3766","issn":["2377-3766","2377-3774"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics and Automation Letters","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1109/lra.2024.3477090","is_oa":true,"landing_page_url":"https://doi.org/10.1109/lra.2024.3477090","pdf_url":null,"source":{"id":"https://openalex.org/S4210169774","display_name":"IEEE Robotics and Automation Letters","issn_l":"2377-3766","issn":["2377-3766","2377-3774"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics and Automation Letters","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":59,"referenced_works":["https://openalex.org/W2126011115","https://openalex.org/W2625366777","https://openalex.org/W2805661097","https://openalex.org/W3021013305","https://openalex.org/W3043330754","https://openalex.org/W3047062112","https://openalex.org/W3117529144","https://openalex.org/W3128894241","https://openalex.org/W3131210852","https://openalex.org/W3135889196","https://openalex.org/W4220879577","https://openalex.org/W4224912544","https://openalex.org/W4301141993","https://openalex.org/W4309130018","https://openalex.org/W4312424618","https://openalex.org/W4313563140","https://openalex.org/W4364322170","https://openalex.org/W4379377990","https://openalex.org/W4383097638","https://openalex.org/W4383108457","https://openalex.org/W4383108852","https://openalex.org/W4383109478","https://openalex.org/W4383172072","https://openalex.org/W4385482474","https://openalex.org/W4386251907","https://openalex.org/W4386320380","https://openalex.org/W4388660746","https://openalex.org/W4389523832","https://openalex.org/W4389665359","https://openalex.org/W4401194599","https://openalex.org/W4401415075","https://openalex.org/W4402716288","https://openalex.org/W6798184267","https://openalex.org/W6809509765","https://openalex.org/W6810640255","https://openalex.org/W6811467201","https://openalex.org/W6839928859","https://openalex.org/W6843759960","https://openalex.org/W6846242362","https://openalex.org/W6846474256","https://openalex.org/W6846770573","https://openalex.org/W6847224448","https://openalex.org/W6847725737","https://openalex.org/W6848398109","https://openalex.org/W6850072970","https://openalex.org/W6854555012","https://openalex.org/W6854738657","https://openalex.org/W6854929498","https://openalex.org/W6856817465","https://openalex.org/W6857062732","https://openalex.org/W6857220666","https://openalex.org/W6857384549","https://openalex.org/W6857423575","https://openalex.org/W6857479877","https://openalex.org/W6857835921","https://openalex.org/W6858376594","https://openalex.org/W6861319346","https://openalex.org/W6862193169","https://openalex.org/W6863270433"],"related_works":["https://openalex.org/W4233452137","https://openalex.org/W4254857216","https://openalex.org/W4231626925","https://openalex.org/W2962829499","https://openalex.org/W2767518918","https://openalex.org/W2131444840","https://openalex.org/W2414929284","https://openalex.org/W192849109","https://openalex.org/W3196817267","https://openalex.org/W1976600725"],"abstract_inverted_index":{"We":[0],"introduce":[1],"a":[2,6,65],"pipeline":[3],"that":[4,33],"enhances":[5],"general-purpose":[7],"Vision":[8],"Language":[9],"Model,":[10],"GPT-4V(ision),":[11],"to":[12,47,94,134],"facilitate":[13],"one-shot":[14,137],"visual":[15],"teaching":[16],"for":[17,105,119],"robotic":[18],"manipulation.":[19],"This":[20,101],"system":[21],"analyzes":[22],"videos":[23,46],"of":[24,51,97,108,146,154],"humans":[25],"performing":[26],"tasks":[27],"and":[28,53,73,89,99,115],"outputs":[29],"executable":[30],"robot":[31,120],"programs":[32],"incorporate":[34],"insights":[35],"into":[36,64],"affordances.":[37],"The":[38],"process":[39],"begins":[40],"with":[41],"GPT-4":[42,149],"V":[43],"analyzing":[44],"the":[45,76,80,106,127,152,159],"obtain":[48],"textual":[49],"explanations":[50],"environmental":[52],"action":[54],"details.":[55],"A":[56],"GPT-4-based":[57],"task":[58,67,77],"planner":[59],"then":[60],"encodes":[61],"these":[62],"details":[63],"symbolic":[66],"plan.":[68],"Subsequently,":[69],"vision":[70],"systems":[71],"spatially":[72],"temporally":[74],"ground":[75],"plan":[78],"in":[79,130,148],"videos\u2014objects":[81],"are":[82,92],"identified":[83],"using":[84],"an":[85],"open-vocabulary":[86],"object":[87],"detector,":[88],"hand-object":[90],"interactions":[91],"analyzed":[93],"pinpoint":[95],"moments":[96],"grasping":[98],"releasing.":[100],"spatiotemporal":[102],"grounding":[103],"allows":[104],"gathering":[107],"affordance":[109],"information":[110],"(e.g.,":[111],"grasp":[112],"types,":[113],"waypoints,":[114],"body":[116],"postures)":[117],"critical":[118],"execution.":[121],"Experiments":[122],"across":[123],"various":[124],"scenarios":[125],"demonstrate":[126],"method's":[128],"efficacy":[129],"enabling":[131],"real":[132],"robots":[133],"operate":[135],"from":[136],"human":[138,156],"demonstrations.":[139],"Meanwhile,":[140],"quantitative":[141],"tests":[142],"have":[143],"revealed":[144],"instances":[145],"hallucination":[147],"V,":[150],"highlighting":[151],"importance":[153],"incorporating":[155],"supervision":[157],"within":[158],"pipeline.":[160]},"counts_by_year":[{"year":2026,"cited_by_count":12},{"year":2025,"cited_by_count":43},{"year":2024,"cited_by_count":9}],"updated_date":"2026-07-30T17:31:21.811387","created_date":"2024-10-10T00:00:00"}