{"id":"https://openalex.org/W4413925997","doi":"https://doi.org/10.1109/icra55743.2025.11127585","title":"A Real-to-Sim-to-Real Approach to Robotic Manipulation with VLM-Generated Iterative Keypoint Rewards","display_name":"A Real-to-Sim-to-Real Approach to Robotic Manipulation with VLM-Generated Iterative Keypoint Rewards","publication_year":2025,"publication_date":"2025-05-19","ids":{"openalex":"https://openalex.org/W4413925997","doi":"https://doi.org/10.1109/icra55743.2025.11127585"},"language":"en","primary_location":{"id":"doi:10.1109/icra55743.2025.11127585","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icra55743.2025.11127585","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037632381","display_name":"Shivansh Patel","orcid":null},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Shivansh Patel","raw_affiliation_strings":["University of Illinois at Urbana-Champaign"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113552658","display_name":"Xian-Yong Yin","orcid":"https://orcid.org/0009-0004-7204-4659"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xinchen Yin","raw_affiliation_strings":["University of Illinois at Urbana-Champaign"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104521370","display_name":"Wenlong Huang","orcid":"https://orcid.org/0000-0003-3611-5654"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wenlong Huang","raw_affiliation_strings":["Stanford University"],"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103058368","display_name":"Shubham Garg","orcid":"https://orcid.org/0000-0002-8089-8732"},"institutions":[{"id":"https://openalex.org/I4210089985","display_name":"Amazon (Germany)","ror":"https://ror.org/00b9ktm87","country_code":"DE","type":"company","lineage":["https://openalex.org/I1311688040","https://openalex.org/I4210089985"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Shubham Garg","raw_affiliation_strings":["Amazon"],"affiliations":[{"raw_affiliation_string":"Amazon","institution_ids":["https://openalex.org/I4210089985"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050088104","display_name":"Hooshang Nayyeri","orcid":"https://orcid.org/0000-0001-8242-9983"},"institutions":[{"id":"https://openalex.org/I4210089985","display_name":"Amazon (Germany)","ror":"https://ror.org/00b9ktm87","country_code":"DE","type":"company","lineage":["https://openalex.org/I1311688040","https://openalex.org/I4210089985"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Hooshang Nayyeri","raw_affiliation_strings":["Amazon"],"affiliations":[{"raw_affiliation_string":"Amazon","institution_ids":["https://openalex.org/I4210089985"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100450462","display_name":"Li Fei-Fei","orcid":"https://orcid.org/0000-0002-7481-0810"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li Fei-Fei","raw_affiliation_strings":["Stanford University"],"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005340983","display_name":"Svetlana Lazebnik","orcid":null},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Svetlana Lazebnik","raw_affiliation_strings":["University of Illinois at Urbana-Champaign"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100340050","display_name":"Yunzhu Li","orcid":"https://orcid.org/0000-0002-1111-2150"},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yunzhu Li","raw_affiliation_strings":["Columbia University"],"affiliations":[{"raw_affiliation_string":"Columbia University","institution_ids":["https://openalex.org/I78577930"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5037632381"],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":null,"apc_paid":null,"fwci":8.5433,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.97907871,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"8258","last_page":"8266"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9804999828338623,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9804999828338623,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10191","display_name":"Robotics and Sensor-Based Localization","score":0.9695000052452087,"subfield":{"id":"https://openalex.org/subfields/2202","display_name":"Aerospace Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10586","display_name":"Robotic Path Planning Algorithms","score":0.9588000178337097,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5614348649978638},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.47023341059684753},{"id":"https://openalex.org/keywords/iterative-method","display_name":"Iterative method","score":0.4301515519618988},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.39741212129592896},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.21066701412200928}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5614348649978638},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47023341059684753},{"id":"https://openalex.org/C159694833","wikidata":"https://www.wikidata.org/wiki/Q2321565","display_name":"Iterative method","level":2,"score":0.4301515519618988},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.39741212129592896},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.21066701412200928}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icra55743.2025.11127585","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icra55743.2025.11127585","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1277750737","display_name":null,"funder_award_id":"HR0011-24-9-0430","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"}],"funders":[{"id":"https://openalex.org/F4320316620","display_name":"Amazon Catalyst","ror":"https://ror.org/04mv4n011"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320332196","display_name":"Sony Electronics","ror":"https://ror.org/05k91zb11"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":36,"referenced_works":["https://openalex.org/W2002553951","https://openalex.org/W2575705757","https://openalex.org/W2604698059","https://openalex.org/W2605102758","https://openalex.org/W2767050701","https://openalex.org/W2962899390","https://openalex.org/W2963184939","https://openalex.org/W2963390419","https://openalex.org/W2963411833","https://openalex.org/W2968116426","https://openalex.org/W3093922502","https://openalex.org/W3114446492","https://openalex.org/W3175254947","https://openalex.org/W4221167977","https://openalex.org/W4230159453","https://openalex.org/W4313160783","https://openalex.org/W4382366145","https://openalex.org/W4385430564","https://openalex.org/W4385430679","https://openalex.org/W4386066287","https://openalex.org/W4386215566","https://openalex.org/W4390873331","https://openalex.org/W4390873395","https://openalex.org/W4391335180","https://openalex.org/W4401415119","https://openalex.org/W4401417251","https://openalex.org/W4402353986","https://openalex.org/W4402354046","https://openalex.org/W4402354151","https://openalex.org/W4402354170","https://openalex.org/W4402716288","https://openalex.org/W4402733576","https://openalex.org/W4402754209","https://openalex.org/W4402781665","https://openalex.org/W4403888374","https://openalex.org/W4405786528"],"related_works":["https://openalex.org/W2772917594","https://openalex.org/W2036807459","https://openalex.org/W2058170566","https://openalex.org/W2755342338","https://openalex.org/W2166024367","https://openalex.org/W3116076068","https://openalex.org/W2229312674","https://openalex.org/W2951359407","https://openalex.org/W2079911747","https://openalex.org/W1969923398"],"abstract_inverted_index":{"Task":[0],"specification":[1],"for":[2,56],"robotic":[3],"manipulation":[4,58],"in":[5,70,106,160,167],"open-world":[6],"environments":[7,169],"is":[8],"challenging,":[9],"requiring":[10],"flexible":[11],"and":[12,20,51,63,73,97,108,141,151],"adaptive":[13],"objectives":[14],"that":[15,38],"align":[16],"with":[17],"human":[18],"intentions":[19],"can":[21],"evolve":[22],"through":[23,170],"iterative":[24,171],"feedback.":[25],"We":[26,102],"introduce":[27],"Iterative":[28],"Keypoint":[29],"Reward":[30],"(IKER),":[31],"a":[32,41,75,127],"visually":[33],"grounded,":[34],"Python-based":[35],"reward":[36,54,76,172],"function":[37,77],"serves":[39],"as":[40],"dynamic":[42,168],"task":[43,146],"specification.":[44],"Our":[45,130],"framework":[46],"leverages":[47],"VLMs":[48],"to":[49,113,163],"generate":[50,74],"refine":[52],"these":[53,80],"functions":[55],"multi-step":[57,145,165],"tasks.":[59],"Given":[60],"RGB-D":[61],"observations":[62],"free-form":[64],"language":[65],"instructions,":[66],"we":[67],"sample":[68],"keypoints":[69],"the":[71,85,94,110,124],"scene":[72],"conditioned":[78],"on":[79,84],"keypoints.":[81],"IKER":[82],"operates":[83],"spatial":[86],"relationships":[87],"between":[88],"keypoints,":[89],"leveraging":[90],"commonsense":[91],"priors":[92],"about":[93],"desired":[95],"behaviors,":[96],"enabling":[98,161],"precise":[99],"SE(3)":[100],"control.":[101],"reconstruct":[103],"real-world":[104],"scenes":[105],"simulation":[107],"use":[109],"generated":[111],"rewards":[112],"train":[114],"reinforcement":[115],"learning":[116],"(RL)":[117],"policies,":[118],"which":[119],"are":[120],"then":[121],"deployed":[122],"into":[123],"real":[125],"world-forming":[126],"real-to-sim-to-real":[128],"loop.":[129],"approach":[131],"demonstrates":[132],"notable":[133],"capabilities":[134],"across":[135],"diverse":[136],"scenarios,":[137],"including":[138],"both":[139],"prehensile":[140],"non-prehensile":[142],"tasks,":[143],"showcasing":[144],"execution,":[147],"spontaneous":[148],"error":[149],"recovery,":[150],"on-the-fly":[152],"strategy":[153],"adjustments.":[154],"The":[155],"results":[156],"highlight":[157],"IKER's":[158],"effectiveness":[159],"robots":[162],"perform":[164],"tasks":[166],"shaping.":[173],"Project":[174],"Page:":[175],"https://iker-robot.github.io/":[176]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":5}],"updated_date":"2026-03-28T08:17:26.163206","created_date":"2025-10-10T00:00:00"}