{"id":"https://openalex.org/W4381894891","doi":"https://doi.org/10.1109/lcsys.2023.3288931","title":"Actor\u2013Critic or Critic\u2013Actor? A Tale of Two Time Scales","display_name":"Actor\u2013Critic or Critic\u2013Actor? A Tale of Two Time Scales","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4381894891","doi":"https://doi.org/10.1109/lcsys.2023.3288931"},"language":"en","primary_location":{"id":"doi:10.1109/lcsys.2023.3288931","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lcsys.2023.3288931","pdf_url":null,"source":{"id":"https://openalex.org/S4306422535","display_name":"IEEE Control Systems Letters","issn_l":"2475-1456","issn":["2475-1456"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Control Systems Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5038163398","display_name":"Shalabh Bhatnagar","orcid":"https://orcid.org/0000-0001-7644-3914"},"institutions":[{"id":"https://openalex.org/I59270414","display_name":"Indian Institute of Science Bangalore","ror":"https://ror.org/04dese585","country_code":"IN","type":"education","lineage":["https://openalex.org/I59270414"]}],"countries":["IN"],"is_corresponding":true,"raw_author_name":"Shalabh Bhatnagar","raw_affiliation_strings":["Department of Computer Science and Automation, Indian Institute of Science, Bengaluru, India"],"raw_orcid":"https://orcid.org/0000-0001-7644-3914","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Automation, Indian Institute of Science, Bengaluru, India","institution_ids":["https://openalex.org/I59270414"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018541798","display_name":"Vivek S. Borkar","orcid":"https://orcid.org/0000-0003-0756-5402"},"institutions":[{"id":"https://openalex.org/I162827531","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152","country_code":"IN","type":"education","lineage":["https://openalex.org/I162827531"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Vivek S. Borkar","raw_affiliation_strings":["Department of Electrical Engineering, Indian Institute of Technology Bombay, Mumbai, India"],"raw_orcid":"https://orcid.org/0000-0003-0756-5402","affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering, Indian Institute of Technology Bombay, Mumbai, India","institution_ids":["https://openalex.org/I162827531"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5011056251","display_name":"Soumyajit Guin","orcid":"https://orcid.org/0000-0003-2327-3549"},"institutions":[{"id":"https://openalex.org/I59270414","display_name":"Indian Institute of Science Bangalore","ror":"https://ror.org/04dese585","country_code":"IN","type":"education","lineage":["https://openalex.org/I59270414"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Soumyajit Guin","raw_affiliation_strings":["Department of Computer Science and Automation, Indian Institute of Science, Bengaluru, India"],"raw_orcid":"https://orcid.org/0000-0003-2327-3549","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Automation, Indian Institute of Science, Bengaluru, India","institution_ids":["https://openalex.org/I59270414"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5038163398"],"corresponding_institution_ids":["https://openalex.org/I59270414"],"apc_list":null,"apc_paid":null,"fwci":1.1929,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.82788494,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":"7","issue":null,"first_page":"2671","last_page":"2676"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.9657999873161316,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9641000032424927,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.6166567802429199},{"id":"https://openalex.org/keywords/stochastic-approximation","display_name":"Stochastic approximation","score":0.5920075178146362},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5765922665596008},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5687779188156128},{"id":"https://openalex.org/keywords/bellman-equation","display_name":"Bellman equation","score":0.547224223613739},{"id":"https://openalex.org/keywords/nonlinear-system","display_name":"Nonlinear system","score":0.5356950163841248},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5022993087768555},{"id":"https://openalex.org/keywords/function-approximation","display_name":"Function approximation","score":0.4698547124862671},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.4674801528453827},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.45241284370422363},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.44321537017822266},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.3654051423072815},{"id":"https://openalex.org/keywords/applied-mathematics","display_name":"Applied mathematics","score":0.33785948157310486},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2706086039543152},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.1950998306274414},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.13729935884475708},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.07794344425201416},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.07056277990341187}],"concepts":[{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.6166567802429199},{"id":"https://openalex.org/C55479107","wikidata":"https://www.wikidata.org/wiki/Q97663916","display_name":"Stochastic approximation","level":3,"score":0.5920075178146362},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5765922665596008},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5687779188156128},{"id":"https://openalex.org/C14646407","wikidata":"https://www.wikidata.org/wiki/Q1430750","display_name":"Bellman equation","level":2,"score":0.547224223613739},{"id":"https://openalex.org/C158622935","wikidata":"https://www.wikidata.org/wiki/Q660848","display_name":"Nonlinear system","level":2,"score":0.5356950163841248},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5022993087768555},{"id":"https://openalex.org/C91873725","wikidata":"https://www.wikidata.org/wiki/Q3445816","display_name":"Function approximation","level":3,"score":0.4698547124862671},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4674801528453827},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.45241284370422363},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.44321537017822266},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3654051423072815},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.33785948157310486},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2706086039543152},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.1950998306274414},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.13729935884475708},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.07794344425201416},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.07056277990341187},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/lcsys.2023.3288931","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lcsys.2023.3288931","pdf_url":null,"source":{"id":"https://openalex.org/S4306422535","display_name":"IEEE Control Systems Letters","issn_l":"2475-1456","issn":["2475-1456"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Control Systems Letters","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320320721","display_name":"Council of Scientific and Industrial Research, India","ror":"https://ror.org/021wm7p51"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W594357522","https://openalex.org/W2009303086","https://openalex.org/W2080631849","https://openalex.org/W2082261506","https://openalex.org/W2091565802","https://openalex.org/W2094387729","https://openalex.org/W2118458590","https://openalex.org/W2139418546","https://openalex.org/W2155027007","https://openalex.org/W2194349390","https://openalex.org/W3041202696","https://openalex.org/W4298023569","https://openalex.org/W4381894891","https://openalex.org/W6683204974","https://openalex.org/W6846611518"],"related_works":["https://openalex.org/W1501190258","https://openalex.org/W4256087190","https://openalex.org/W4240668504","https://openalex.org/W3119329039","https://openalex.org/W2788366696","https://openalex.org/W4306703873","https://openalex.org/W2964123095","https://openalex.org/W1553308207","https://openalex.org/W2569146624","https://openalex.org/W3188865574"],"abstract_inverted_index":{"We":[0,34,53],"revisit":[1],"the":[2,39,61],"standard":[3],"formulation":[4],"of":[5,38,57,90],"tabular":[6],"actor-critic":[7,87],"algorithm":[8,82],"as":[9],"a":[10,20,27,50,55],"two":[11,62],"time-scale":[12,22],"stochastic":[13],"approximation":[14,68],"with":[15,64,86],"value":[16,46],"function":[17,67,74],"computed":[18,25],"on":[19,26,84],"faster":[21],"and":[23,48,59,65,72,76,93],"policy":[24,32],"slower":[28],"time-scale.":[29],"This":[30],"emulates":[31],"iteration.":[33],"observe":[35,77],"that":[36,78],"reversal":[37],"time":[40],"scales":[41],"will":[42],"in":[43,88],"fact":[44],"emulate":[45],"iteration":[47],"is":[49],"legitimate":[51],"algorithm.":[52],"provide":[54],"proof":[56],"convergence":[58],"compare":[60],"empirically":[63],"without":[66],"(with":[69],"both":[70,91],"linear":[71],"nonlinear":[73],"approximators)":[75],"our":[79],"proposed":[80],"critic-actor":[81],"performs":[83],"par":[85],"terms":[89],"accuracy":[92],"computational":[94],"effort.":[95]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}