{"id":"https://openalex.org/W2068923975","doi":"https://doi.org/10.1109/tamd.2012.2205924","title":"Model-Free Reinforcement Learning of Impedance Control in Stochastic Environments","display_name":"Model-Free Reinforcement Learning of Impedance Control in Stochastic Environments","publication_year":2012,"publication_date":"2012-06-28","ids":{"openalex":"https://openalex.org/W2068923975","doi":"https://doi.org/10.1109/tamd.2012.2205924","mag":"2068923975"},"language":"en","primary_location":{"id":"doi:10.1109/tamd.2012.2205924","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tamd.2012.2205924","pdf_url":null,"source":{"id":"https://openalex.org/S108448409","display_name":"IEEE Transactions on Autonomous Mental Development","issn_l":"1943-0604","issn":["1943-0604","1943-0612"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Autonomous Mental Development","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5017689065","display_name":"Freek Stulp","orcid":"https://orcid.org/0000-0001-9555-9517"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]},{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I201181511","display_name":"\u00c9cole Nationale Sup\u00e9rieure de Techniques Avanc\u00e9es","ror":"https://ror.org/0309cs235","country_code":"FR","type":"education","lineage":["https://openalex.org/I201181511","https://openalex.org/I4210145102"]},{"id":"https://openalex.org/I4210131512","display_name":"Centre Inria de l'universit\u00e9 de Bordeaux","ror":"https://ror.org/03tjcj052","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1326498283","https://openalex.org/I4210131512"]}],"countries":["FR","US"],"is_corresponding":false,"raw_author_name":"Freek Stulp","raw_affiliation_strings":["Cognitive Robotics Department, \u00c9cole Nationale Sup\u00e9rieure de Techniques Avanc\u00e9es-ParisTech, Paris, France","Computational Learning and Motor Control Laboratory, University of Southern California, Los Angeles, CA, USA","INRIA Bordeaux Sud-Ouest, FLOWERS Research Team, Talence, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cognitive Robotics Department, \u00c9cole Nationale Sup\u00e9rieure de Techniques Avanc\u00e9es-ParisTech, Paris, France","institution_ids":["https://openalex.org/I201181511"]},{"raw_affiliation_string":"Computational Learning and Motor Control Laboratory, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]},{"raw_affiliation_string":"INRIA Bordeaux Sud-Ouest, FLOWERS Research Team, Talence, France","institution_ids":["https://openalex.org/I1326498283","https://openalex.org/I4210131512"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077279214","display_name":"Jonas Buchli","orcid":"https://orcid.org/0000-0001-7494-8492"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]},{"id":"https://openalex.org/I30771326","display_name":"Italian Institute of Technology","ror":"https://ror.org/042t93s57","country_code":"IT","type":"facility","lineage":["https://openalex.org/I30771326"]}],"countries":["IT","US"],"is_corresponding":false,"raw_author_name":"Jonas Buchli","raw_affiliation_strings":["Computational Learning and Motor Control Laboratory, University of Southern California, Los Angeles, CA, USA","Department of Advanced Robotics, Istituto Italiano di Tecnologia, Genoa, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computational Learning and Motor Control Laboratory, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]},{"raw_affiliation_string":"Department of Advanced Robotics, Istituto Italiano di Tecnologia, Genoa, Italy","institution_ids":["https://openalex.org/I30771326"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050568796","display_name":"Alice Ellmer","orcid":null},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alice Ellmer","raw_affiliation_strings":["Computational Learning andMotor Control Laboratory, University of Southern California, Los Angeles, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computational Learning andMotor Control Laboratory, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082553836","display_name":"Michael Mistry","orcid":"https://orcid.org/0000-0003-3979-922X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Michael Mistry","raw_affiliation_strings":["School of Computer Science, University of Binningham, Birmingham, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science, University of Binningham, Birmingham, UK","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044505993","display_name":"Evangelos A. Theodorou","orcid":"https://orcid.org/0000-0002-0834-5738"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]},{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Evangelos A. Theodorou","raw_affiliation_strings":["Computational Learning andMotor Control Laboratory, University of Southern California, Los Angeles, CA, USA","Computer Science and Engineering Department, University of Washington, Seattle, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computational Learning andMotor Control Laboratory, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]},{"raw_affiliation_string":"Computer Science and Engineering Department, University of Washington, Seattle, USA","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I58610484"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5107392143","display_name":"Stefan Schaal","orcid":null},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stefan Schaal","raw_affiliation_strings":["Computational Learning and Motor Control Laboratory, University of Southern California, Los Angeles, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computational Learning and Motor Control Laboratory, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.6422,"has_fulltext":false,"cited_by_count":64,"citation_normalized_percentile":{"value":0.65919537,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"4","issue":"4","first_page":"330","last_page":"341"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10982","display_name":"Motor Control and Adaptation","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10982","display_name":"Motor Control and Adaptation","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10784","display_name":"Muscle activation and electromyography studies","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.7502118945121765},{"id":"https://openalex.org/keywords/impedance-control","display_name":"Impedance control","score":0.6612109541893005},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6140449047088623},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5939874053001404},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5296454429626465},{"id":"https://openalex.org/keywords/electrical-impedance","display_name":"Electrical impedance","score":0.45506995916366577},{"id":"https://openalex.org/keywords/robotics","display_name":"Robotics","score":0.4363248348236084},{"id":"https://openalex.org/keywords/simulation","display_name":"Simulation","score":0.3622402846813202},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.19009685516357422}],"concepts":[{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.7502118945121765},{"id":"https://openalex.org/C2777984285","wikidata":"https://www.wikidata.org/wiki/Q17098134","display_name":"Impedance control","level":3,"score":0.6612109541893005},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6140449047088623},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5939874053001404},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5296454429626465},{"id":"https://openalex.org/C17829176","wikidata":"https://www.wikidata.org/wiki/Q179043","display_name":"Electrical impedance","level":2,"score":0.45506995916366577},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.4363248348236084},{"id":"https://openalex.org/C44154836","wikidata":"https://www.wikidata.org/wiki/Q45045","display_name":"Simulation","level":1,"score":0.3622402846813202},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.19009685516357422},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/tamd.2012.2205924","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tamd.2012.2205924","pdf_url":null,"source":{"id":"https://openalex.org/S108448409","display_name":"IEEE Transactions on Autonomous Mental Development","issn_l":"1943-0604","issn":["1943-0604","1943-0612"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Autonomous Mental Development","raw_type":"journal-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.699.4370","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.699.4370","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.freekstulp.net/publications/pdfs/stulp12modelfree.pdf","raw_type":"text"},{"id":"pmh:oai:pure.mpg.de:item_1703323","is_oa":false,"landing_page_url":"http://hdl.handle.net/11858/00-001M-0000-000E-EA59-9","pdf_url":null,"source":{"id":"https://openalex.org/S4306400654","display_name":"MPG.PuRe (Max Planck Society)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I149899117","host_organization_name":"Max Planck Society","host_organization_lineage":["https://openalex.org/I149899117"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Transactions on Autonomous Mental Development","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Affordable and clean energy","score":0.8999999761581421,"id":"https://metadata.un.org/sdg/7"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W1586251222","https://openalex.org/W1873164297","https://openalex.org/W1922715780","https://openalex.org/W1925816294","https://openalex.org/W1957096125","https://openalex.org/W1967377907","https://openalex.org/W1976606095","https://openalex.org/W1986201898","https://openalex.org/W1987710890","https://openalex.org/W2005417378","https://openalex.org/W2026659355","https://openalex.org/W2027795063","https://openalex.org/W2030268430","https://openalex.org/W2050162940","https://openalex.org/W2071444114","https://openalex.org/W2110304639","https://openalex.org/W2112036188","https://openalex.org/W2113265921","https://openalex.org/W2127341404","https://openalex.org/W2132622533","https://openalex.org/W2133932631","https://openalex.org/W2147828974","https://openalex.org/W2156321584","https://openalex.org/W2168650125","https://openalex.org/W2169136412","https://openalex.org/W2171080843","https://openalex.org/W2990651410","https://openalex.org/W3005581722","https://openalex.org/W6640290305","https://openalex.org/W6679365036","https://openalex.org/W6683111245"],"related_works":["https://openalex.org/W4306904969","https://openalex.org/W2138720691","https://openalex.org/W4362501864","https://openalex.org/W1548357495","https://openalex.org/W2314048082","https://openalex.org/W1968487248","https://openalex.org/W2097006611","https://openalex.org/W4295916778","https://openalex.org/W2112995122","https://openalex.org/W2165087119"],"abstract_inverted_index":{"For":[0],"humans":[1,60,73],"and":[2,14,30,38,87,101,140,156],"robots,":[3,191],"variable":[4],"impedance":[5,26,57,80,98],"control":[6,58],"is":[7],"an":[8],"essential":[9],"component":[10],"for":[11,190,198],"ensuring":[12],"robust":[13],"safe":[15],"physical":[16],"interaction":[17],"with":[18,126],"the":[19,114,145,150,154],"environment.":[20],"Humans":[21],"learn":[22,108],"to":[23,27,77,81,112,144,166],"adapt":[24,78],"their":[25,79,97],"specific":[28],"tasks":[29],"environments;":[31],"a":[32,62,109,119,162,175,201],"capability":[33],"which":[34],"we":[35,41,50],"continually":[36],"develop":[37],"improve":[39],"until":[40],"are":[42,93,105],"well":[43],"into":[44],"our":[45,127],"twenties.":[46],"In":[47],"this":[48],"article,":[49],"reproduce":[51],"functionally":[52],"interesting":[53],"aspects":[54],"of":[55,177],"learning":[56,130,167],"in":[59,68],"on":[61],"simulated":[63,121],"robot":[64,122,155,202],"platform.":[65],"As":[66],"demonstrated":[67],"numerous":[69],"force":[70,142],"field":[71],"tasks,":[72],"combine":[74],"two":[75],"strategies":[76],"perturbations,":[82],"thereby":[83],"minimizing":[84],"position":[85],"error":[86],"energy":[88],"consumption:":[89],"1)":[90],"if":[91,103],"perturbations":[92,104],"unpredictable,":[94],"subjects":[95,107],"increase":[96],"through":[99],"cocontraction;":[100],"2)":[102],"predictable,":[106],"feed-forward":[110],"command":[111],"offset":[113],"perturbation.":[115],"We":[116,148],"show":[117,149],"how":[118],"7-DOF":[120],"demonstrates":[123],"similar":[124],"behavior":[125],"model-free":[128],"reinforcement":[129],"algorithm":[131],"PI":[132],"<sup":[133],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[134],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">2</sup>":[135],",":[136],"by":[137],"applying":[138],"deterministic":[139],"stochastic":[141],"fields":[143],"robot's":[146],"end-effector.":[147],"qualitative":[151],"similarity":[152],"between":[153],"human":[157],"movements.":[158],"Our":[159],"results":[160],"provide":[161],"biologically":[163],"plausible":[164],"approach":[165],"appropriate":[168],"impedances":[169],"purely":[170],"from":[171],"experience,":[172],"without":[173],"requiring":[174,184],"model":[176],"either":[178],"body":[179],"or":[180],"environment":[181,200],"dynamics.":[182],"Not":[183],"models":[185,194],"also":[186],"facilitates":[187],"autonomous":[188],"development":[189],"as":[192],"prespecified":[193],"cannot":[195],"be":[196],"provided":[197],"each":[199],"might":[203],"encounter.":[204]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":9},{"year":2022,"cited_by_count":12},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":8},{"year":2018,"cited_by_count":5},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":3},{"year":2015,"cited_by_count":2},{"year":2014,"cited_by_count":2},{"year":2012,"cited_by_count":1}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}