{"id":"https://openalex.org/W4285597472","doi":"https://doi.org/10.48550/arxiv.2207.06572","title":"i-Sim2Real: Reinforcement Learning of Robotic Policies in Tight Human-Robot Interaction Loops","display_name":"i-Sim2Real: Reinforcement Learning of Robotic Policies in Tight Human-Robot Interaction Loops","publication_year":2022,"publication_date":"2022-07-14","ids":{"openalex":"https://openalex.org/W4285597472","doi":"https://doi.org/10.48550/arxiv.2207.06572"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2207.06572","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2207.06572","pdf_url":"https://arxiv.org/pdf/2207.06572","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2207.06572","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011850830","display_name":"Saminda W. Abeyruwan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Abeyruwan, Saminda","raw_affiliation_strings":["Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002723909","display_name":"Laura Graesser","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Graesser, Laura","raw_affiliation_strings":["Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078539163","display_name":"David B. D\u2019Ambrosio","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"D'Ambrosio, David B.","raw_affiliation_strings":["Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071640510","display_name":"Avi Singh","orcid":"https://orcid.org/0009-0002-7874-6999","observed_orcids":["https://orcid.org/0009-0002-7874-6999"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Singh, Avi","raw_affiliation_strings":["Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039532136","display_name":"Anish Shankar","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shankar, Anish","raw_affiliation_strings":["Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043765267","display_name":"Alex Bewley","orcid":"https://orcid.org/0000-0002-8428-9264","observed_orcids":["https://orcid.org/0000-0002-8428-9264"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bewley, Alex","raw_affiliation_strings":["Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Jain, Deepali","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jain, Deepali","raw_affiliation_strings":["Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Choromanski, Krzysztof","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Choromanski, Krzysztof","raw_affiliation_strings":["Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":null,"display_name":"Sanketi, Pannag R.","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sanketi, Pannag R.","raw_affiliation_strings":["Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robotics at Google {saminda , lauragraesser , ddambro , singhavi ,","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1291425158"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":14,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.007899999618530273,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.0005000000237487257,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sim-to-real-transfer","display_name":"sim-to-real transfer","score":0.9940000176429749},{"id":"https://openalex.org/keywords/human-robot-interaction","display_name":"human-robot interaction","score":0.9539999961853027},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"reinforcement learning","score":0.5730000138282776},{"id":"https://openalex.org/keywords/iterative-simulation","display_name":"iterative simulation","score":0.33500000834465027},{"id":"https://openalex.org/keywords/human-behavior-modeling","display_name":"human behavior modeling","score":0.9359999895095825},{"id":"https://openalex.org/keywords/evolutionary-search","display_name":"evolutionary search","score":0.9190000295639038},{"id":"https://openalex.org/keywords/cooperative-play","display_name":"cooperative play","score":0.5230000019073486},{"id":"https://openalex.org/keywords/industrial-robotic-arm","display_name":"industrial robot arm","score":0.9100000262260437}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8163436055183411},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.7412623167037964},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.7301563024520874},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6886508464813232},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5740487575531006},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.5539450645446777},{"id":"https://openalex.org/C45235069","wikidata":"https://www.wikidata.org/wiki/Q278425","display_name":"Table (database)","level":2,"score":0.493269145488739},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4776840806007385},{"id":"https://openalex.org/C19766214","wikidata":"https://www.wikidata.org/wiki/Q4880688","display_name":"Behavior-based robotics","level":4,"score":0.46075209975242615},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.43746793270111084},{"id":"https://openalex.org/C44154836","wikidata":"https://www.wikidata.org/wiki/Q45045","display_name":"Simulation","level":1,"score":0.3504185676574707},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.3003193736076355},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1386280357837677},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2207.06572","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2207.06572","pdf_url":"https://arxiv.org/pdf/2207.06572","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2207.06572","doi":"https://doi.org/10.48550/arxiv.2207.06572","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2207.06572","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2207.06572","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2207.06572","pdf_url":"https://arxiv.org/pdf/2207.06572","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4285597472.pdf","grobid_xml":"https://content.openalex.org/works/W4285597472.grobid-xml"},"referenced_works_count":11,"referenced_works":["https://openalex.org/W2562736583","https://openalex.org/W2596367596","https://openalex.org/W2902690953","https://openalex.org/W2963184939","https://openalex.org/W2963669336","https://openalex.org/W3037785630","https://openalex.org/W3130568950","https://openalex.org/W3151513620","https://openalex.org/W3152331843","https://openalex.org/W3198020111","https://openalex.org/W4226111152"],"related_works":["https://openalex.org/W4397028883","https://openalex.org/W4315640913","https://openalex.org/W7135139717","https://openalex.org/W6910456940","https://openalex.org/W4405927045","https://openalex.org/W7213981188","https://openalex.org/W4383109464","https://openalex.org/W3008492644","https://openalex.org/W4414051096","https://openalex.org/W2899460553"],"abstract_inverted_index":{"Sim-to-real":[0],"transfer":[1,33],"is":[2,49,58,80,113,196,211,248],"a":[3,81,92,96,110,118,132,168,183,201,212,230],"powerful":[4],"paradigm":[5],"for":[6,193,204,229,234,271],"robotic":[7,35,67,186,245],"reinforcement":[8],"learning.":[9],"The":[10],"ability":[11],"to":[12,59,65,88,101,115,126,197,221,224,250,279,283],"train":[13,66],"policies":[14,36,68],"in":[15,31,105,142,146,295],"simulation":[16,64,106,143],"enables":[17],"safe":[18],"exploration":[19],"and":[20,83,138,144,158,266],"large-scale":[21],"data":[22],"collection":[23],"quickly":[24,223],"at":[25,72,268],"low":[26],"cost.":[27],"However,":[28],"prior":[29],"works":[30],"sim-to-real":[32,285],"of":[34,63,91,135,260,273,292],"typically":[37],"do":[38],"not":[39],"involve":[40],"any":[41],"human-robot":[42,237],"interaction":[43],"because":[44],"accurately":[45],"simulating":[46],"human":[47,93,103,136,155,202,256],"behavior":[48,104,137,156],"an":[50,243],"open":[51],"problem.":[52],"In":[53,150],"this":[54],"work,":[55],"our":[56,180,293],"goal":[57],"leverage":[60],"the":[61,147,154,159,191,194,218,284],"power":[62],"that":[69,112,216,247],"are":[70,161,277],"proficient":[71],"interacting":[73,94],"with":[74,95,117,200,255],"humans":[75],"upon":[76],"deployment.":[77],"But":[78],"there":[79],"chicken":[82],"egg":[84],"problem":[85],"--":[86],"how":[87],"gather":[89],"examples":[90],"physical":[97],"robot":[98,111,195],"so":[99],"as":[100,205,207],"model":[102,134,157],"without":[107],"already":[108],"having":[109],"able":[114,249],"interact":[116],"human?":[119],"Our":[120],"proposed":[121],"method,":[122],"Iterative-Sim-to-Real":[123],"(i-S2R),":[124],"attempts":[125],"address":[127],"this.":[128],"i-S2R":[129],"bootstraps":[130],"from":[131],"simple":[133],"alternates":[139],"between":[140],"training":[141,165],"deploying":[145],"real":[148,184],"world.":[149],"each":[151,225],"iteration,":[152],"both":[153],"policy":[160],"refined.":[162],"For":[163,290],"all":[164],"we":[166],"apply":[167],"new":[169],"evolutionary":[170],"search":[171],"algorithm":[172],"called":[173],"Blackbox":[174],"Gradient":[175],"Sensing":[176],"(BGS).":[177],"We":[178,239],"evaluate":[179],"method":[181],"on":[182,236,242,264],"world":[185],"table":[187,253],"tennis":[188,210,254],"setting,":[189],"where":[190],"objective":[192],"play":[198,252],"cooperatively":[199,251],"player":[203],"long":[206],"possible.":[208],"Table":[209],"high-speed,":[213],"dynamic":[214],"task":[215],"requires":[217],"two":[219],"players":[220],"react":[222],"other's":[226],"moves,":[227],"making":[228],"challenging":[231],"test":[232],"bed":[233],"research":[235],"interaction.":[238],"present":[240],"results":[241],"industrial":[244],"arm":[246],"players,":[257,274],"achieving":[258],"rallies":[259],"22":[261],"successive":[262],"hits":[263],"average":[265],"150":[267],"best.":[269],"Further,":[270],"80%":[272],"rally":[275],"lengths":[276],"70%":[278],"175%":[280],"longer":[281],"compared":[282],"plus":[286],"fine-tuning":[287],"(S2R+FT)":[288],"baseline.":[289],"videos":[291],"system":[294],"action,":[296],"please":[297],"see":[298],"https://sites.google.com/view/is2r.":[299]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":10}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2022-07-16T00:00:00"}