{"id":"https://openalex.org/W4391021192","doi":"https://doi.org/10.1109/cdc49753.2023.10384223","title":"Non-Stationary Policy Learning for Multi-Timescale Multi-Agent Reinforcement Learning","display_name":"Non-Stationary Policy Learning for Multi-Timescale Multi-Agent Reinforcement Learning","publication_year":2023,"publication_date":"2023-12-13","ids":{"openalex":"https://openalex.org/W4391021192","doi":"https://doi.org/10.1109/cdc49753.2023.10384223"},"language":"en","primary_location":{"id":"doi:10.1109/cdc49753.2023.10384223","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cdc49753.2023.10384223","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 62nd IEEE Conference on Decision and Control (CDC)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074789327","display_name":"Patrick Emami","orcid":"https://orcid.org/0000-0001-7846-5578"},"institutions":[{"id":"https://openalex.org/I1297288678","display_name":"National Laboratory of the Rockies","ror":"https://ror.org/036266993","country_code":"US","type":"facility","lineage":["https://openalex.org/I1297288678","https://openalex.org/I1330989302","https://openalex.org/I2800842121"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Patrick Emami","raw_affiliation_strings":["National Renewable Energy Laboratory,Golden,CO,USA,80401"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Renewable Energy Laboratory,Golden,CO,USA,80401","institution_ids":["https://openalex.org/I1297288678"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100362472","display_name":"Xiangyu Zhang","orcid":"https://orcid.org/0000-0003-4857-2318"},"institutions":[{"id":"https://openalex.org/I1297288678","display_name":"National Laboratory of the Rockies","ror":"https://ror.org/036266993","country_code":"US","type":"facility","lineage":["https://openalex.org/I1297288678","https://openalex.org/I1330989302","https://openalex.org/I2800842121"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiangyu Zhang","raw_affiliation_strings":["National Renewable Energy Laboratory,Golden,CO,USA,80401"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Renewable Energy Laboratory,Golden,CO,USA,80401","institution_ids":["https://openalex.org/I1297288678"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000527374","display_name":"David Biagioni","orcid":"https://orcid.org/0000-0001-6140-1957"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"David Biagioni","raw_affiliation_strings":["Maplewell Energy,Broomfield,CO,USA,80021"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Maplewell Energy,Broomfield,CO,USA,80021","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008068890","display_name":"Ahmed S. Zamzam","orcid":"https://orcid.org/0000-0002-1175-2801"},"institutions":[{"id":"https://openalex.org/I1297288678","display_name":"National Laboratory of the Rockies","ror":"https://ror.org/036266993","country_code":"US","type":"facility","lineage":["https://openalex.org/I1297288678","https://openalex.org/I1330989302","https://openalex.org/I2800842121"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ahmed S. Zamzam","raw_affiliation_strings":["National Renewable Energy Laboratory,Golden,CO,USA,80401"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Renewable Energy Laboratory,Golden,CO,USA,80401","institution_ids":["https://openalex.org/I1297288678"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.6526,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.76414585,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"2372","last_page":"2378"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.9879999756813049,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11206","display_name":"Model Reduction and Neural Networks","score":0.9817000031471252,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.893193244934082},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6933989524841309},{"id":"https://openalex.org/keywords/policy-learning","display_name":"Policy learning","score":0.5717607736587524},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.48705700039863586},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.4746987223625183},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.45398133993148804},{"id":"https://openalex.org/keywords/phase","display_name":"Phase (matter)","score":0.42746254801750183},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3257303833961487},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.07479932904243469}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.893193244934082},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6933989524841309},{"id":"https://openalex.org/C2779436431","wikidata":"https://www.wikidata.org/wiki/Q30672407","display_name":"Policy learning","level":2,"score":0.5717607736587524},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.48705700039863586},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.4746987223625183},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45398133993148804},{"id":"https://openalex.org/C44280652","wikidata":"https://www.wikidata.org/wiki/Q104837","display_name":"Phase (matter)","level":2,"score":0.42746254801750183},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3257303833961487},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.07479932904243469},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/cdc49753.2023.10384223","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cdc49753.2023.10384223","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 62nd IEEE Conference on Decision and Control (CDC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W1521003796","https://openalex.org/W2058356438","https://openalex.org/W2088956500","https://openalex.org/W2121111990","https://openalex.org/W2166533447","https://openalex.org/W2611706523","https://openalex.org/W2617547828","https://openalex.org/W2740377041","https://openalex.org/W2788643836","https://openalex.org/W2949464762","https://openalex.org/W2954559956","https://openalex.org/W3104819573","https://openalex.org/W3119215358","https://openalex.org/W3157870065","https://openalex.org/W4225009817","https://openalex.org/W4307006423","https://openalex.org/W4310974370","https://openalex.org/W6631137000","https://openalex.org/W6738796088","https://openalex.org/W6742529797","https://openalex.org/W6746684068","https://openalex.org/W6785899988"],"related_works":["https://openalex.org/W1585007175","https://openalex.org/W2382521049","https://openalex.org/W4306904969","https://openalex.org/W2144385241","https://openalex.org/W2138720691","https://openalex.org/W4362501864","https://openalex.org/W4380318855","https://openalex.org/W2031695474","https://openalex.org/W2024136090","https://openalex.org/W4300101996"],"abstract_inverted_index":{"In":[0,11,77],"multi-timescale":[1,60,132],"multi-agent":[2,75,97],"reinforcement":[3],"learning":[4,56],"(MARL),":[5],"agents":[6],"interact":[7],"across":[8],"different":[9],"timescales.":[10],"general,":[12],"policies":[13,28,58,133],"for":[14,55,59,124],"time-dependent":[15],"behaviors,":[16],"such":[17,101],"as":[18],"those":[19],"induced":[20],"by":[21,39,88,94],"multiple":[22,89],"timescales,":[23],"are":[24],"non-stationary.":[25],"Learning":[26],"non-stationary":[27,57],"is":[29,134],"challenging":[30],"and":[31,72,113,139],"typically":[32],"requires":[33],"sophisticated":[34],"or":[35],"inefficient":[36],"algorithms.":[37],"Motivated":[38],"the":[40,83,111],"prevalence":[41],"of":[42,85],"this":[43],"control":[44],"problem":[45],"in":[46],"real-world":[47],"complex":[48],"systems,":[49],"we":[50,79,103],"introduce":[51],"a":[52,95,105,137],"simple":[53],"framework":[54],"MARL.":[61],"Our":[62],"approach":[63],"uses":[64],"available":[65],"information":[66],"about":[67],"agent":[68],"timescales":[69,90],"to":[70,129],"define":[71],"learn":[73,100,131],"periodic":[74,96],"policies.":[76],"detail,":[78],"theoretically":[80],"demonstrate":[81],"that":[82,109],"effects":[84],"non-stationarity":[86],"introduced":[87],"can":[91],"be":[92],"learned":[93],"policy.":[98],"To":[99],"policies,":[102],"propose":[104],"policy":[106],"gradient":[107],"algorithm":[108],"parameterizes":[110],"actor":[112],"critic":[114],"with":[115],"phase-functioned":[116],"neural":[117],"networks,":[118],"which":[119],"provide":[120],"an":[121],"inductive":[122],"bias":[123],"periodicity.":[125],"The":[126],"framework's":[127],"ability":[128],"effectively":[130],"validated":[135],"on":[136],"gridworld":[138],"building":[140],"energy":[141],"management":[142],"environment.":[143]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}