{"id":"https://openalex.org/W2767696060","doi":"https://doi.org/10.1109/tcyb.2017.2761841","title":"Off-Policy Reinforcement Learning: Optimal Operational Control for Two-Time-Scale Industrial Processes","display_name":"Off-Policy Reinforcement Learning: Optimal Operational Control for Two-Time-Scale Industrial Processes","publication_year":2017,"publication_date":"2017-11-07","ids":{"openalex":"https://openalex.org/W2767696060","doi":"https://doi.org/10.1109/tcyb.2017.2761841","mag":"2767696060","pmid":"https://pubmed.ncbi.nlm.nih.gov/29125464"},"language":"en","primary_location":{"id":"doi:10.1109/tcyb.2017.2761841","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcyb.2017.2761841","pdf_url":null,"source":{"id":"https://openalex.org/S4210191041","display_name":"IEEE Transactions on Cybernetics","issn_l":"2168-2267","issn":["2168-2267","2168-2275"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cybernetics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070599005","display_name":"Jinna Li","orcid":"https://orcid.org/0000-0001-9985-6308"},"institutions":[{"id":"https://openalex.org/I48780066","display_name":"Shenyang University of Chemical Technology","ror":"https://ror.org/03dbpdh75","country_code":"CN","type":"education","lineage":["https://openalex.org/I48780066"]},{"id":"https://openalex.org/I4391767858","display_name":"State Key Laboratory of Synthetical Automation for Process Industries","ror":"https://ror.org/0380ng272","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391767858","https://openalex.org/I9224756"]},{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Jinna Li","raw_affiliation_strings":["College of Information Engineering, Shenyang University of Chemical Technology, Liaoning, China","State Key Laboratory of Synthetical Automation for Process Industries and the International Joint Research Laboratory of Integrated Automation, Northeastern University, Shenyang, China"],"affiliations":[{"raw_affiliation_string":"College of Information Engineering, Shenyang University of Chemical Technology, Liaoning, China","institution_ids":["https://openalex.org/I48780066"]},{"raw_affiliation_string":"State Key Laboratory of Synthetical Automation for Process Industries and the International Joint Research Laboratory of Integrated Automation, Northeastern University, Shenyang, China","institution_ids":["https://openalex.org/I9224756","https://openalex.org/I4391767858"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089224471","display_name":"Bahare Kiumarsi","orcid":"https://orcid.org/0000-0002-9701-8375"},"institutions":[{"id":"https://openalex.org/I189196454","display_name":"The University of Texas at Arlington","ror":"https://ror.org/019kgqr73","country_code":"US","type":"education","lineage":["https://openalex.org/I189196454"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bahare Kiumarsi","raw_affiliation_strings":["UTA Research Institute, University of Texas at Arlington, Arlington, TX, USA"],"affiliations":[{"raw_affiliation_string":"UTA Research Institute, University of Texas at Arlington, Arlington, TX, USA","institution_ids":["https://openalex.org/I189196454"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042520521","display_name":"Tianyou Chai","orcid":"https://orcid.org/0000-0002-4623-1483"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]},{"id":"https://openalex.org/I4391767858","display_name":"State Key Laboratory of Synthetical Automation for Process Industries","ror":"https://ror.org/0380ng272","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391767858","https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianyou Chai","raw_affiliation_strings":["State Key Laboratory of Synthetical Automation for Process Industries and the International Joint Research Laboratory of Integrated Automation, Northeastern University, Shenyang, China"],"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Synthetical Automation for Process Industries and the International Joint Research Laboratory of Integrated Automation, Northeastern University, Shenyang, China","institution_ids":["https://openalex.org/I9224756","https://openalex.org/I4391767858"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016137188","display_name":"Frank L. Lewis","orcid":"https://orcid.org/0000-0003-4074-1615"},"institutions":[{"id":"https://openalex.org/I189196454","display_name":"The University of Texas at Arlington","ror":"https://ror.org/019kgqr73","country_code":"US","type":"education","lineage":["https://openalex.org/I189196454"]},{"id":"https://openalex.org/I4391767858","display_name":"State Key Laboratory of Synthetical Automation for Process Industries","ror":"https://ror.org/0380ng272","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391767858","https://openalex.org/I9224756"]},{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Frank L. Lewis","raw_affiliation_strings":["State Key Laboratory of Synthetical Automation for Process Industries and the International Joint Research Laboratory of Integrated Automation, Northeastern University, Shenyang, China","UTA Research Institute, University of Texas at Arlington, Arlington, TX, USA"],"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Synthetical Automation for Process Industries and the International Joint Research Laboratory of Integrated Automation, Northeastern University, Shenyang, China","institution_ids":["https://openalex.org/I9224756","https://openalex.org/I4391767858"]},{"raw_affiliation_string":"UTA Research Institute, University of Texas at Arlington, Arlington, TX, USA","institution_ids":["https://openalex.org/I189196454"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5033960992","display_name":"Jialu Fan","orcid":"https://orcid.org/0000-0001-7585-1166"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]},{"id":"https://openalex.org/I4391767858","display_name":"State Key Laboratory of Synthetical Automation for Process Industries","ror":"https://ror.org/0380ng272","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391767858","https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jialu Fan","raw_affiliation_strings":["State Key Laboratory of Synthetical Automation for Process Industries and the International Joint Research Laboratory of Integrated Automation, Northeastern University, Shenyang, China"],"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Synthetical Automation for Process Industries and the International Joint Research Laboratory of Integrated Automation, Northeastern University, Shenyang, China","institution_ids":["https://openalex.org/I9224756","https://openalex.org/I4391767858"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5070599005"],"corresponding_institution_ids":["https://openalex.org/I4391767858","https://openalex.org/I48780066","https://openalex.org/I9224756"],"apc_list":null,"apc_paid":null,"fwci":6.5863,"has_fulltext":false,"cited_by_count":69,"citation_normalized_percentile":{"value":0.97383423,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"47","issue":"12","first_page":"4547","last_page":"4558"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.9959999918937683,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.9959999918937683,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14011","display_name":"Elevator Systems and Control","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10791","display_name":"Advanced Control Systems Optimization","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7736590504646301},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.622117280960083},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5905295014381409},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5537204742431641},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5128472447395325},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.4311125874519348},{"id":"https://openalex.org/keywords/optimal-control","display_name":"Optimal control","score":0.41238754987716675},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.4024519920349121},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1989603340625763},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.16324421763420105}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7736590504646301},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.622117280960083},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5905295014381409},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5537204742431641},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5128472447395325},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.4311125874519348},{"id":"https://openalex.org/C91575142","wikidata":"https://www.wikidata.org/wiki/Q1971426","display_name":"Optimal control","level":2,"score":0.41238754987716675},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4024519920349121},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1989603340625763},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.16324421763420105},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tcyb.2017.2761841","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcyb.2017.2761841","pdf_url":null,"source":{"id":"https://openalex.org/S4210191041","display_name":"IEEE Transactions on Cybernetics","issn_l":"2168-2267","issn":["2168-2267","2168-2275"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cybernetics","raw_type":"journal-article"},{"id":"pmid:29125464","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/29125464","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on cybernetics","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.5799999833106995,"display_name":"Industry, innovation and infrastructure"}],"awards":[{"id":"https://openalex.org/G1460833335","display_name":null,"funder_award_id":"61533015","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3951694549","display_name":null,"funder_award_id":"N160804001","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"},{"id":"https://openalex.org/G5156944241","display_name":null,"funder_award_id":"61525302","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5409562055","display_name":null,"funder_award_id":"B08015","funder_id":"https://openalex.org/F4320327912","funder_display_name":"Higher Education Discipline Innovation Project"},{"id":"https://openalex.org/G7272473876","display_name":null,"funder_award_id":"61590922","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G74425646","display_name":null,"funder_award_id":"61673280","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7583496911","display_name":null,"funder_award_id":"61333012","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8875637860","display_name":null,"funder_award_id":"61104093","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320327912","display_name":"Higher Education Discipline Innovation Project","ror":null},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W658381347","https://openalex.org/W1567906773","https://openalex.org/W1579118835","https://openalex.org/W1614417283","https://openalex.org/W1965628919","https://openalex.org/W1972044154","https://openalex.org/W1972243698","https://openalex.org/W1976231238","https://openalex.org/W1978531916","https://openalex.org/W2006491884","https://openalex.org/W2025527701","https://openalex.org/W2025752438","https://openalex.org/W2025963754","https://openalex.org/W2026915918","https://openalex.org/W2029661056","https://openalex.org/W2039091159","https://openalex.org/W2040871222","https://openalex.org/W2056664593","https://openalex.org/W2064380515","https://openalex.org/W2084693097","https://openalex.org/W2088404810","https://openalex.org/W2112103082","https://openalex.org/W2117680746","https://openalex.org/W2118569209","https://openalex.org/W2138155072","https://openalex.org/W2171065122","https://openalex.org/W2329769476","https://openalex.org/W2339745223","https://openalex.org/W2473060649","https://openalex.org/W2580629550","https://openalex.org/W2752501126","https://openalex.org/W3149446627","https://openalex.org/W4295133295","https://openalex.org/W6677406069"],"related_works":["https://openalex.org/W4362501864","https://openalex.org/W4306904969","https://openalex.org/W4380318855","https://openalex.org/W2138720691","https://openalex.org/W2031695474","https://openalex.org/W2586732548","https://openalex.org/W3049728571","https://openalex.org/W20361778","https://openalex.org/W2024136090","https://openalex.org/W2964765435"],"abstract_inverted_index":{"Industrial":[0],"flow":[1],"lines":[2],"are":[3,70],"composed":[4],"of":[5,37,126],"unit":[6,50,88],"processes":[7,41],"operating":[8],"on":[9],"a":[10,23,30,35,55,66,73,92,112],"fast":[11,56],"time":[12,25,68],"scale":[13,69],"and":[14,59],"performance":[15],"measurements":[16],"known":[17],"as":[18],"operational":[19,62,76],"indices":[20],"measured":[21,108],"at":[22,65],"slower":[24],"scale.":[26],"This":[27],"paper":[28],"presents":[29],"model-free":[31],"optimal":[32,75,103],"solution":[33],"to":[34,81,100,122],"class":[36],"two":[38],"time-scale":[39],"industrial":[40,89,119],"using":[42,106],"off-policy":[43,95],"reinforcement":[44],"learning":[45],"(RL).":[46],"First,":[47],"the":[48,60,84,87,102,124,127],"lower-layer":[49],"process":[51,121],"control":[52,77],"loop":[53],"with":[54],"sampling":[57],"period":[58],"upper-layer":[61],"index":[63],"dynamics":[64],"slow":[67],"modeled.":[71],"Second,":[72],"general":[74],"problem":[78],"is":[79,98,115],"formulated":[80],"optimally":[82],"prescribe":[83],"set-points":[85,104],"for":[86,117],"process.":[90],"Then,":[91],"zero-sum":[93],"game":[94],"RL":[96],"algorithm":[97],"developed":[99],"find":[101],"by":[105],"data":[107],"in":[109],"real-time.":[110],"Finally,":[111],"simulation":[113],"experiment":[114],"employed":[116],"an":[118],"flotation":[120],"show":[123],"effectiveness":[125],"proposed":[128],"method.":[129]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":9},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":9},{"year":2020,"cited_by_count":10},{"year":2019,"cited_by_count":11},{"year":2018,"cited_by_count":7},{"year":2017,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}