{"id":"https://openalex.org/W4416747877","doi":"https://doi.org/10.1109/iros60139.2025.11246918","title":"Real-World Offline Reinforcement Learning from Vision Language Model Feedback","display_name":"Real-World Offline Reinforcement Learning from Vision Language Model Feedback","publication_year":2025,"publication_date":"2025-10-19","ids":{"openalex":"https://openalex.org/W4416747877","doi":"https://doi.org/10.1109/iros60139.2025.11246918"},"language":null,"primary_location":{"id":"doi:10.1109/iros60139.2025.11246918","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iros60139.2025.11246918","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5104231157","display_name":"Sreyas Venkataraman","orcid":null},"institutions":[{"id":"https://openalex.org/I145894827","display_name":"Indian Institute of Technology Kharagpur","ror":"https://ror.org/03w5sq511","country_code":"IN","type":"education","lineage":["https://openalex.org/I145894827"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Sreyas Venkataraman","raw_affiliation_strings":["Indian Institute of Technology,Kharagpur"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indian Institute of Technology,Kharagpur","institution_ids":["https://openalex.org/I145894827"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100374840","display_name":"Yufei Wang","orcid":"https://orcid.org/0000-0003-3016-6389"},"institutions":[{"id":"https://openalex.org/I145894827","display_name":"Indian Institute of Technology Kharagpur","ror":"https://ror.org/03w5sq511","country_code":"IN","type":"education","lineage":["https://openalex.org/I145894827"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Yufei Wang","raw_affiliation_strings":["Indian Institute of Technology,Kharagpur"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indian Institute of Technology,Kharagpur","institution_ids":["https://openalex.org/I145894827"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100420910","display_name":"Ziyu Wang","orcid":"https://orcid.org/0000-0001-9534-3134"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziyu Wang","raw_affiliation_strings":["Tsinghua University,IIIS"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,IIIS","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120049793","display_name":"Navin Sriram Ravie","orcid":null},"institutions":[{"id":"https://openalex.org/I145894827","display_name":"Indian Institute of Technology Kharagpur","ror":"https://ror.org/03w5sq511","country_code":"IN","type":"education","lineage":["https://openalex.org/I145894827"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Navin Sriram Ravie","raw_affiliation_strings":["Indian Institute of Technology,Kharagpur"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indian Institute of Technology,Kharagpur","institution_ids":["https://openalex.org/I145894827"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075365855","display_name":"Zackory Erickson","orcid":"https://orcid.org/0000-0002-6760-6213"},"institutions":[{"id":"https://openalex.org/I145894827","display_name":"Indian Institute of Technology Kharagpur","ror":"https://ror.org/03w5sq511","country_code":"IN","type":"education","lineage":["https://openalex.org/I145894827"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Zackory Erickson","raw_affiliation_strings":["Indian Institute of Technology,Kharagpur"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indian Institute of Technology,Kharagpur","institution_ids":["https://openalex.org/I145894827"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037048516","display_name":"David Held","orcid":"https://orcid.org/0000-0003-0537-1508"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Held","raw_affiliation_strings":["Carnegie Mellon University,Robotics Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University,Robotics Institute","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"13452","last_page":"13459"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.42500001192092896,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.42500001192092896,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.3840999901294708,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.04989999905228615,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.75},{"id":"https://openalex.org/keywords/offline-learning","display_name":"Offline learning","score":0.7264999747276306},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6317999958992004},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5888000130653381},{"id":"https://openalex.org/keywords/online-and-offline","display_name":"Online and offline","score":0.49129998683929443},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.48489999771118164},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.4099999964237213}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7922999858856201},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.75},{"id":"https://openalex.org/C2780490138","wikidata":"https://www.wikidata.org/wiki/Q7079636","display_name":"Offline learning","level":3,"score":0.7264999747276306},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6854000091552734},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6317999958992004},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5888000130653381},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5605000257492065},{"id":"https://openalex.org/C2780102126","wikidata":"https://www.wikidata.org/wiki/Q10928179","display_name":"Online and offline","level":2,"score":0.49129998683929443},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.48489999771118164},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.4099999964237213},{"id":"https://openalex.org/C2776639384","wikidata":"https://www.wikidata.org/wiki/Q840396","display_name":"Ideal (ethics)","level":2,"score":0.3605000078678131},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.359499990940094},{"id":"https://openalex.org/C2986087404","wikidata":"https://www.wikidata.org/wiki/Q15946010","display_name":"Online learning","level":2,"score":0.3465000092983246},{"id":"https://openalex.org/C2777851325","wikidata":"https://www.wikidata.org/wiki/Q7094102","display_name":"Online model","level":2,"score":0.3443000018596649},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2637999951839447},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.2572999894618988},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.25609999895095825},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.2506999969482422}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/iros60139.2025.11246918","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iros60139.2025.11246918","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332178","display_name":"National Institute of Standards and Technology","ror":"https://ror.org/05xpvk416"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":10,"referenced_works":["https://openalex.org/W2031571562","https://openalex.org/W2051228319","https://openalex.org/W2091565802","https://openalex.org/W2194775991","https://openalex.org/W2754517384","https://openalex.org/W2763110165","https://openalex.org/W4385403811","https://openalex.org/W4385430480","https://openalex.org/W4392667322","https://openalex.org/W4402354045"],"related_works":[],"abstract_inverted_index":{"Offline":[0],"reinforcement":[1],"learning":[2,6,161,209],"can":[3,216],"enable":[4],"policy":[5,114,208],"from":[7,98,210],"pre-collected,":[8],"sub-optimal":[9,147,212],"datasets":[10,94],"without":[11],"online":[12,26],"interactions.":[13],"This":[14],"makes":[15],"it":[16],"ideal":[17],"for":[18,92],"real-world":[19,130],"robots":[20],"and":[21,34,82,102,150,181,184,192,207],"safety-critical":[22],"scenarios,":[23],"where":[24,134],"collecting":[25],"data":[27],"or":[28],"expert":[29],"demonstrations":[30],"is":[31,45],"slow,":[32],"costly,":[33],"risky.":[35],"However,":[36],"most":[37],"existing":[38],"offline":[39,93,116,148,213],"RL":[40,117],"works":[41],"assume":[42],"the":[43,49,70,107,119,124,154,177],"dataset":[44],"already":[46],"labeled":[47],"with":[48,118],"task":[50],"rewards,":[51],"a":[52,84,99,103,113,128,138,142,146,199],"process":[53],"that":[54,87,202],"often":[55],"requires":[56],"significant":[57],"human":[58],"effort,":[59],"especially":[60],"when":[61],"ground-truth":[62],"states":[63],"are":[64],"hard":[65],"to":[66,127,157,162],"ascertain":[67],"(e.g.,":[68],"in":[69,173],"real-world).":[71],"In":[72,195],"this":[73],"paper,":[74],"we":[75,135,152,197],"build":[76],"on":[77,145,219],"prior":[78],"work,":[79],"specifically":[80],"RL-VLM-F,":[81],"propose":[83,198],"novel":[85],"system":[86,201],"automatically":[88],"generates":[89],"reward":[90,139,156,205],"labels":[91],"using":[95,115,141],"preference":[96],"feedback":[97],"vision-language":[100,143],"model":[101,144],"text":[104],"description":[105],"of":[106,179],"task.":[108],"Our":[109,168],"method":[110,169],"then":[111,151],"learns":[112],"reward-labeled":[120],"dataset.":[121],"We":[122],"demonstrate":[123],"system\u2019s":[125],"applicability":[126],"complex":[129],"robot-assisted":[131],"dressing":[132,166],"task,":[133],"first":[136],"learn":[137],"function":[140],"dataset,":[149],"use":[153],"learned":[155],"employ":[158],"Implicit":[159],"Q":[160],"develop":[163],"an":[164],"effective":[165],"policy.":[167],"also":[170],"performs":[171],"well":[172],"simulation":[174],"tasks":[175],"involving":[176],"manipulation":[178],"rigid":[180],"deformable":[182],"objects,":[183],"significantly":[185],"outperforms":[186],"baselines":[187],"such":[188],"as":[189],"behavior":[190],"cloning":[191],"inverse":[193],"RL.":[194],"summary,":[196],"new":[200],"enables":[203],"automatic":[204],"labeling":[206],"unlabeled,":[211],"datasets.":[214],"Videos":[215],"be":[217],"found":[218],"our":[220],"project":[221],"website<sup":[222],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[223],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">1</sup>.":[224]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-11-28T00:00:00"}