{"id":"https://openalex.org/W1966086707","doi":"https://doi.org/10.1109/tsmcb.2011.2170565","title":"Efficient Model Learning Methods for Actor\u2013Critic Control","display_name":"Efficient Model Learning Methods for Actor\u2013Critic Control","publication_year":2011,"publication_date":"2011-12-07","ids":{"openalex":"https://openalex.org/W1966086707","doi":"https://doi.org/10.1109/tsmcb.2011.2170565","mag":"1966086707","pmid":"https://pubmed.ncbi.nlm.nih.gov/22156998"},"language":"en","primary_location":{"id":"doi:10.1109/tsmcb.2011.2170565","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tsmcb.2011.2170565","pdf_url":null,"source":{"id":"https://openalex.org/S4210170378","display_name":"IEEE Transactions on Systems Man and Cybernetics Part B (Cybernetics)","issn_l":"1083-4419","issn":["1083-4419","1941-0492"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Systems, Man, and Cybernetics, Part B (Cybernetics)","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028459687","display_name":"I. Grondman","orcid":null},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"I. Grondman","raw_affiliation_strings":["Delft Center for Systems and Control, Delft University of Technology, Delft, The Netherlands. i.grondman@tudelft.nl","Delft Center for Systems and Control, Delft University of Technology, Delft, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Delft Center for Systems and Control, Delft University of Technology, Delft, The Netherlands. i.grondman@tudelft.nl","institution_ids":["https://openalex.org/I98358874"]},{"raw_affiliation_string":"Delft Center for Systems and Control, Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008086562","display_name":"Maarten Vaandrager","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"M. Vaandrager","raw_affiliation_strings":["Plotprojects, Amsterdam, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Plotprojects, Amsterdam, Netherlands","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058935509","display_name":"Lucian Bu\u015foniu","orcid":"https://orcid.org/0000-0001-8017-1296"},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I158333966","display_name":"Technical University of Cluj-Napoca","ror":"https://ror.org/03r8nwp71","country_code":"RO","type":"education","lineage":["https://openalex.org/I158333966"]},{"id":"https://openalex.org/I4210110792","display_name":"Centre de Recherche en Automatique de Nancy","ror":"https://ror.org/022r5hc56","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I4210099203","https://openalex.org/I4210100260","https://openalex.org/I4210110792","https://openalex.org/I4210159245","https://openalex.org/I90183372"]},{"id":"https://openalex.org/I90183372","display_name":"Universit\u00e9 de Lorraine","ror":"https://ror.org/04vfs2w97","country_code":"FR","type":"education","lineage":["https://openalex.org/I90183372"]}],"countries":["FR","RO"],"is_corresponding":false,"raw_author_name":"L. Busoniu","raw_affiliation_strings":["CNRS, Research Center for Automatic Control, University of Lorraine, Nancy, France","Department of Automation, Technical University of Cluj Napoca, Cluj-Napoca, Romania"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CNRS, Research Center for Automatic Control, University of Lorraine, Nancy, France","institution_ids":["https://openalex.org/I4210110792","https://openalex.org/I90183372","https://openalex.org/I1294671590"]},{"raw_affiliation_string":"Department of Automation, Technical University of Cluj Napoca, Cluj-Napoca, Romania","institution_ids":["https://openalex.org/I158333966"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084264842","display_name":"Robert Babu\u0161ka","orcid":"https://orcid.org/0000-0001-9578-8598"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"R. Babuska","raw_affiliation_strings":["Delft Center for Systems and Control, Delft University of Technology, Delft, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Delft Center for Systems and Control, Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031675619","display_name":"E. Schuitema","orcid":null},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"E. Schuitema","raw_affiliation_strings":["Delft Biorobotics Laboratory, Faculty of Mechanical Engineering, Delft University of Technology, Delft, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Delft Biorobotics Laboratory, Faculty of Mechanical Engineering, Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.0393,"has_fulltext":false,"cited_by_count":119,"citation_normalized_percentile":{"value":0.9607552,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":"42","issue":"3","first_page":"591","last_page":"602"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11206","display_name":"Model Reduction and Neural Networks","score":0.9745000004768372,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.718849241733551},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6258417367935181},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.569571852684021},{"id":"https://openalex.org/keywords/inverted-pendulum","display_name":"Inverted pendulum","score":0.5442723035812378},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5040837526321411},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.4711662232875824},{"id":"https://openalex.org/keywords/temporal-difference-learning","display_name":"Temporal difference learning","score":0.4582788646221161},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4397360682487488},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.4111081063747406},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.4109049439430237},{"id":"https://openalex.org/keywords/nonlinear-system","display_name":"Nonlinear system","score":0.14039385318756104}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.718849241733551},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6258417367935181},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.569571852684021},{"id":"https://openalex.org/C192921069","wikidata":"https://www.wikidata.org/wiki/Q550134","display_name":"Inverted pendulum","level":3,"score":0.5442723035812378},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5040837526321411},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4711662232875824},{"id":"https://openalex.org/C196340769","wikidata":"https://www.wikidata.org/wiki/Q7698910","display_name":"Temporal difference learning","level":3,"score":0.4582788646221161},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4397360682487488},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.4111081063747406},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4109049439430237},{"id":"https://openalex.org/C158622935","wikidata":"https://www.wikidata.org/wiki/Q660848","display_name":"Nonlinear system","level":2,"score":0.14039385318756104},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003198","descriptor_name":"Computer Simulation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003198","descriptor_name":"Computer Simulation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003198","descriptor_name":"Computer Simulation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003661","descriptor_name":"Decision Support Techniques","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003661","descriptor_name":"Decision Support Techniques","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003661","descriptor_name":"Decision Support Techniques","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D008962","descriptor_name":"Models, Theoretical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D008962","descriptor_name":"Models, Theoretical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D008962","descriptor_name":"Models, Theoretical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D010363","descriptor_name":"Pattern Recognition, Automated","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D010363","descriptor_name":"Pattern Recognition, Automated","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D010363","descriptor_name":"Pattern Recognition, Automated","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false}],"locations_count":2,"locations":[{"id":"doi:10.1109/tsmcb.2011.2170565","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tsmcb.2011.2170565","pdf_url":null,"source":{"id":"https://openalex.org/S4210170378","display_name":"IEEE Transactions on Systems Man and Cybernetics Part B (Cybernetics)","issn_l":"1083-4419","issn":["1083-4419","1941-0492"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Systems, Man, and Cybernetics, Part B (Cybernetics)","raw_type":"journal-article"},{"id":"pmid:22156998","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/22156998","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on systems, man, and cybernetics. Part B, Cybernetics : a publication of the IEEE Systems, Man, and Cybernetics Society","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.6100000143051147}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W1540685400","https://openalex.org/W1592209052","https://openalex.org/W1626155273","https://openalex.org/W1827261456","https://openalex.org/W1949804828","https://openalex.org/W1983523797","https://openalex.org/W1986984034","https://openalex.org/W2009303086","https://openalex.org/W2048226872","https://openalex.org/W2091565802","https://openalex.org/W2092276439","https://openalex.org/W2094364653","https://openalex.org/W2094387729","https://openalex.org/W2121863487","https://openalex.org/W2122496402","https://openalex.org/W2126685977","https://openalex.org/W2143490508","https://openalex.org/W2145830976","https://openalex.org/W2150999967","https://openalex.org/W2153733100","https://openalex.org/W2155027007","https://openalex.org/W2156321584","https://openalex.org/W2170396314","https://openalex.org/W2172968643","https://openalex.org/W2597289420","https://openalex.org/W2764902300","https://openalex.org/W4214717370","https://openalex.org/W4245108548","https://openalex.org/W6632235069","https://openalex.org/W6635668049","https://openalex.org/W6646985475"],"related_works":["https://openalex.org/W2145363145","https://openalex.org/W2341346307","https://openalex.org/W2154399718","https://openalex.org/W4321463377","https://openalex.org/W2768629321","https://openalex.org/W2130711276","https://openalex.org/W4308828368","https://openalex.org/W1528400370","https://openalex.org/W3038962357","https://openalex.org/W2189613824"],"abstract_inverted_index":{"We":[0],"propose":[1],"two":[2,100],"new":[3],"actor-critic":[4,106],"algorithms":[5,10,28],"for":[6,48,60],"reinforcement":[7],"learning.":[8,50],"Both":[9],"use":[11,69],"local":[12],"linear":[13],"regression":[14],"(LLR)":[15],"to":[16,110],"learn":[17,33],"approximations":[18],"of":[19,26,94],"the":[20,27,61,92,95,111,117,124],"functions":[21],"involved.":[22],"A":[23],"crucial":[24],"feature":[25],"is":[29],"that":[30],"they":[31],"also":[32],"a":[34,55,75,80,104],"process":[35,97],"model,":[36],"and":[37,103],"this,":[38],"in":[39,115],"combination":[40],"with":[41],"LLR,":[42],"provides":[43],"an":[44,70],"efficient":[45],"policy":[46],"update":[47,58],"faster":[49,121],"The":[51,64,99],"first":[52],"algorithm":[53,66,107],"uses":[54],"novel":[56,101,118],"model-based":[57],"rule":[59],"actor":[62,72],"parameters.":[63],"second":[65],"does":[67],"not":[68],"explicit":[71],"but":[73],"learns":[74],"reference":[76],"model":[77],"which":[78,84,116],"represents":[79],"desired":[81,85],"behavior,":[82],"from":[83],"control":[86],"actions":[87],"can":[88],"be":[89],"calculated":[90],"using":[91],"inverse":[93],"learned":[96],"model.":[98],"methods":[102,119],"standard":[105,125],"are":[108],"applied":[109],"pendulum":[112],"swing-up":[113],"problem,":[114],"achieve":[120],"learning":[122],"than":[123],"algorithm.":[126]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":21},{"year":2020,"cited_by_count":26},{"year":2019,"cited_by_count":13},{"year":2018,"cited_by_count":12},{"year":2017,"cited_by_count":7},{"year":2016,"cited_by_count":4},{"year":2015,"cited_by_count":6},{"year":2014,"cited_by_count":10},{"year":2013,"cited_by_count":3},{"year":2012,"cited_by_count":1}],"updated_date":"2026-01-13T01:12:25.745995","created_date":"2025-10-10T00:00:00"}