{"id":"https://openalex.org/W4402261296","doi":"https://doi.org/10.23919/acc60939.2024.10644470","title":"Investigating the Effectiveness of Reinforcement Learning in Closed-Loop Systems with Time Delays","display_name":"Investigating the Effectiveness of Reinforcement Learning in Closed-Loop Systems with Time Delays","publication_year":2024,"publication_date":"2024-07-10","ids":{"openalex":"https://openalex.org/W4402261296","doi":"https://doi.org/10.23919/acc60939.2024.10644470"},"language":"en","primary_location":{"id":"doi:10.23919/acc60939.2024.10644470","is_oa":false,"landing_page_url":"https://doi.org/10.23919/acc60939.2024.10644470","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 American Control Conference (ACC)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076706861","display_name":"Moh Kamalul Wafi","orcid":"https://orcid.org/0000-0002-2016-5721"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Moh Kamalul Wafi","raw_affiliation_strings":["Northeastern University,Department of Electrical & Computer Engineering,Boston,MA,USA,02115"],"affiliations":[{"raw_affiliation_string":"Northeastern University,Department of Electrical & Computer Engineering,Boston,MA,USA,02115","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058687251","display_name":"Milad Siami","orcid":"https://orcid.org/0000-0001-7253-4464"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Milad Siami","raw_affiliation_strings":["Northeastern University,Department of Electrical & Computer Engineering,Boston,MA,USA,02115"],"affiliations":[{"raw_affiliation_string":"Northeastern University,Department of Electrical & Computer Engineering,Boston,MA,USA,02115","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074266892","display_name":"Mario Sznaier","orcid":"https://orcid.org/0000-0003-4439-3988"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mario Sznaier","raw_affiliation_strings":["Northeastern University,Department of Electrical & Computer Engineering,Boston,MA,USA,02115"],"affiliations":[{"raw_affiliation_string":"Northeastern University,Department of Electrical & Computer Engineering,Boston,MA,USA,02115","institution_ids":["https://openalex.org/I12912129"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5076706861"],"corresponding_institution_ids":["https://openalex.org/I12912129"],"apc_list":null,"apc_paid":null,"fwci":0.6989,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.70580153,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"4149","last_page":"4154"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14083","display_name":"Extremum Seeking Control Systems","score":0.9786999821662903,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14083","display_name":"Extremum Seeking Control Systems","score":0.9786999821662903,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9771000146865845,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11513","display_name":"stochastic dynamics and bifurcation","score":0.9753999710083008,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7513786554336548},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6639060378074646},{"id":"https://openalex.org/keywords/loop","display_name":"Loop (graph theory)","score":0.5744320750236511},{"id":"https://openalex.org/keywords/closed-loop","display_name":"Closed loop","score":0.49900174140930176},{"id":"https://openalex.org/keywords/control-theory","display_name":"Control theory (sociology)","score":0.4729849100112915},{"id":"https://openalex.org/keywords/reinforcement","display_name":"Reinforcement","score":0.46046683192253113},{"id":"https://openalex.org/keywords/feedback-loop","display_name":"Feedback loop","score":0.4415857791900635},{"id":"https://openalex.org/keywords/control-engineering","display_name":"Control engineering","score":0.35231539607048035},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2504592537879944},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.17280766367912292},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.12904557585716248},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.0855206847190857},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.06073272228240967}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7513786554336548},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6639060378074646},{"id":"https://openalex.org/C184670325","wikidata":"https://www.wikidata.org/wiki/Q512604","display_name":"Loop (graph theory)","level":2,"score":0.5744320750236511},{"id":"https://openalex.org/C3019251811","wikidata":"https://www.wikidata.org/wiki/Q5135346","display_name":"Closed loop","level":2,"score":0.49900174140930176},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.4729849100112915},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.46046683192253113},{"id":"https://openalex.org/C186886427","wikidata":"https://www.wikidata.org/wiki/Q5441213","display_name":"Feedback loop","level":2,"score":0.4415857791900635},{"id":"https://openalex.org/C133731056","wikidata":"https://www.wikidata.org/wiki/Q4917288","display_name":"Control engineering","level":1,"score":0.35231539607048035},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2504592537879944},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.17280766367912292},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.12904557585716248},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0855206847190857},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.06073272228240967},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C66938386","wikidata":"https://www.wikidata.org/wiki/Q633538","display_name":"Structural engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.23919/acc60939.2024.10644470","is_oa":false,"landing_page_url":"https://doi.org/10.23919/acc60939.2024.10644470","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 American Control Conference (ACC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2855079135","display_name":null,"funder_award_id":"N00014-21-1-2431","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G4960767043","display_name":null,"funder_award_id":"2121121,2208182","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5391824508","display_name":null,"funder_award_id":"W911NF-22-2-0001","funder_id":"https://openalex.org/F4320338295","funder_display_name":"Army Research Laboratory"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"},{"id":"https://openalex.org/F4320338295","display_name":"Army Research Laboratory","ror":"https://ror.org/011hc8f90"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W1506119886","https://openalex.org/W1546613882","https://openalex.org/W1588998206","https://openalex.org/W2100970306","https://openalex.org/W2106551534","https://openalex.org/W2117864026","https://openalex.org/W2121863487","https://openalex.org/W2159566498","https://openalex.org/W2529061134","https://openalex.org/W2973229164","https://openalex.org/W3013672175","https://openalex.org/W3142744390","https://openalex.org/W3149836039","https://openalex.org/W3173185086","https://openalex.org/W3190141713","https://openalex.org/W4383824426","https://openalex.org/W4387914438"],"related_works":["https://openalex.org/W2322335058","https://openalex.org/W2259382430","https://openalex.org/W2375499102","https://openalex.org/W2381125525","https://openalex.org/W3084132679","https://openalex.org/W2397460850","https://openalex.org/W4322484979","https://openalex.org/W4301401229","https://openalex.org/W2612895087","https://openalex.org/W2024563306"],"abstract_inverted_index":{"Data-driven":[0],"controllers":[1,73],"have":[2],"gained":[3],"prominence":[4],"in":[5,24,31,123,196],"diverse":[6],"control":[7,99],"applications,":[8],"attributed":[9],"to":[10,16,77,96,128,137,173],"their":[11,32,79],"inherent":[12],"flexibility":[13],"and":[14,48,59,74,108,144,184,191],"adaptability":[15],"complex":[17],"system":[18,57,109,164],"dynamics.":[19],"However,":[20],"managing":[21,104],"time":[22,53,69,106],"delays":[23,35,70,107],"closed-loop":[25],"systems":[26],"remains":[27],"a":[28,98,156,160],"significant":[29],"challenge":[30],"deployment.":[33],"These":[34],"can":[36,55],"arise":[37],"from":[38],"various":[39],"sources,":[40],"such":[41,188],"as":[42,189],"computational":[43],"latency,":[44],"actuator":[45],"reaction":[46],"time,":[47],"communication":[49],"delays.":[50,200],"Unaddressed,":[51],"these":[52],"lags":[54],"induce":[56],"instability":[58],"degrade":[60],"performance.":[61,114],"This":[62],"paper":[63],"rigorously":[64],"analyzes":[65],"the":[66,85,88,134,138,175,181,185,197],"impact":[67],"of":[68,87,102,141,177,199],"on":[71],"data-driven":[72],"introduces":[75],"methodologies":[76],"mitigate":[78],"adverse":[80],"effects.":[81],"Specifically,":[82],"we":[83,154],"explore":[84],"integration":[86],"Smith":[89],"predictor":[90],"with":[91,165],"Deep":[92],"Reinforcement":[93],"Learning":[94],"(SP-DRL)":[95],"formulate":[97],"law":[100],"capable":[101],"effectively":[103],"both":[105],"uncertainties,":[110],"while":[111],"ensuring":[112],"robust":[113],"We":[115],"demonstrate":[116],"that":[117],"this":[118,142],"DRL-based":[119],"framework,":[120],"initially":[121],"trained":[122],"stable":[124],"environments,":[125],"generalizes":[126],"well":[127],"unstable":[129],"systems.":[130],"Our":[131],"investigation":[132],"delineates":[133],"scenarios":[135],"conducive":[136],"successful":[139],"application":[140],"approach":[143],"identifies":[145],"factors":[146],"influencing":[147],"its":[148],"effectiveness.":[149],"To":[150],"substantiate":[151],"our":[152],"findings,":[153],"present":[155],"case":[157],"study":[158],"involving":[159],"first-order":[161],"delayed":[162],"linear":[163],"nonlinear":[166],"actuation":[167],"modules.":[168],"Numerical":[169],"simulations":[170],"are":[171],"employed":[172],"compare":[174],"robustness":[176],"SP-DRL":[178],"scheme":[179],"against":[180],"DRL":[182],"standalone":[183],"classical":[186],"controls,":[187],"PID":[190],"Linear":[192],"Quadratic":[193],"Regulator":[194],"(LQR),":[195],"presence":[198]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2025-12-27T23:08:20.325037","created_date":"2025-10-10T00:00:00"}