{"id":"https://openalex.org/W4294691626","doi":"https://doi.org/10.23919/acc53348.2022.9867291","title":"Convex Programs and Lyapunov Functions for Reinforcement Learning: A Unified Perspective on the Analysis of Value-Based Methods","display_name":"Convex Programs and Lyapunov Functions for Reinforcement Learning: A Unified Perspective on the Analysis of Value-Based Methods","publication_year":2022,"publication_date":"2022-06-08","ids":{"openalex":"https://openalex.org/W4294691626","doi":"https://doi.org/10.23919/acc53348.2022.9867291"},"language":"en","primary_location":{"id":"doi:10.23919/acc53348.2022.9867291","is_oa":false,"landing_page_url":"https://doi.org/10.23919/acc53348.2022.9867291","pdf_url":null,"source":{"id":"https://openalex.org/S4363607732","display_name":"2022 American Control Conference (ACC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 American Control Conference (ACC)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101652354","display_name":"Xingang Guo","orcid":"https://orcid.org/0000-0002-7169-9053"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Xingang Guo","raw_affiliation_strings":["University of Illinois at Urbana-Champaign,Coordinated Science Laboratory (CSL),Department of Electrical and Computer Engineering","Department of Electrical and Computer Engineering, Coordinated Science Laboratory (CSL), University of Illinois at Urbana-Champaign"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign,Coordinated Science Laboratory (CSL),Department of Electrical and Computer Engineering","institution_ids":["https://openalex.org/I157725225"]},{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Coordinated Science Laboratory (CSL), University of Illinois at Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101513622","display_name":"Bin Hu","orcid":"https://orcid.org/0000-0002-5632-7737"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bin Hu","raw_affiliation_strings":["University of Illinois at Urbana-Champaign,Coordinated Science Laboratory (CSL),Department of Electrical and Computer Engineering","Department of Electrical and Computer Engineering, Coordinated Science Laboratory (CSL), University of Illinois at Urbana-Champaign"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign,Coordinated Science Laboratory (CSL),Department of Electrical and Computer Engineering","institution_ids":["https://openalex.org/I157725225"]},{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Coordinated Science Laboratory (CSL), University of Illinois at Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5101652354"],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":null,"apc_paid":null,"fwci":0.1039,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.28355958,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"3317","last_page":"3322"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.989799976348877,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.989799976348877,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10142","display_name":"Formal Methods in Verification","score":0.921999990940094,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10621","display_name":"Gene Regulatory Network Analysis","score":0.9147999882698059,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.7244636416435242},{"id":"https://openalex.org/keywords/lyapunov-function","display_name":"Lyapunov function","score":0.7174420356750488},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6911788582801819},{"id":"https://openalex.org/keywords/bellman-equation","display_name":"Bellman equation","score":0.6736776232719421},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6075028777122498},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.596964418888092},{"id":"https://openalex.org/keywords/convex-optimization","display_name":"Convex optimization","score":0.5663127303123474},{"id":"https://openalex.org/keywords/intersection","display_name":"Intersection (aeronautics)","score":0.5580581426620483},{"id":"https://openalex.org/keywords/convex-analysis","display_name":"Convex analysis","score":0.5251328945159912},{"id":"https://openalex.org/keywords/semidefinite-programming","display_name":"Semidefinite programming","score":0.508098840713501},{"id":"https://openalex.org/keywords/convex-function","display_name":"Convex function","score":0.4423708915710449},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.4262978136539459},{"id":"https://openalex.org/keywords/regular-polygon","display_name":"Regular polygon","score":0.3758299648761749},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.33577761054039},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.31444165110588074},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.18922096490859985},{"id":"https://openalex.org/keywords/nonlinear-system","display_name":"Nonlinear system","score":0.1320616602897644}],"concepts":[{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.7244636416435242},{"id":"https://openalex.org/C60640748","wikidata":"https://www.wikidata.org/wiki/Q2337858","display_name":"Lyapunov function","level":3,"score":0.7174420356750488},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6911788582801819},{"id":"https://openalex.org/C14646407","wikidata":"https://www.wikidata.org/wiki/Q1430750","display_name":"Bellman equation","level":2,"score":0.6736776232719421},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6075028777122498},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.596964418888092},{"id":"https://openalex.org/C157972887","wikidata":"https://www.wikidata.org/wiki/Q463359","display_name":"Convex optimization","level":3,"score":0.5663127303123474},{"id":"https://openalex.org/C64543145","wikidata":"https://www.wikidata.org/wiki/Q162942","display_name":"Intersection (aeronautics)","level":2,"score":0.5580581426620483},{"id":"https://openalex.org/C12108790","wikidata":"https://www.wikidata.org/wiki/Q2234833","display_name":"Convex analysis","level":4,"score":0.5251328945159912},{"id":"https://openalex.org/C101901036","wikidata":"https://www.wikidata.org/wiki/Q2269096","display_name":"Semidefinite programming","level":2,"score":0.508098840713501},{"id":"https://openalex.org/C145446738","wikidata":"https://www.wikidata.org/wiki/Q319913","display_name":"Convex function","level":3,"score":0.4423708915710449},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.4262978136539459},{"id":"https://openalex.org/C112680207","wikidata":"https://www.wikidata.org/wiki/Q714886","display_name":"Regular polygon","level":2,"score":0.3758299648761749},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.33577761054039},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.31444165110588074},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.18922096490859985},{"id":"https://openalex.org/C158622935","wikidata":"https://www.wikidata.org/wiki/Q660848","display_name":"Nonlinear system","level":2,"score":0.1320616602897644},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.23919/acc53348.2022.9867291","is_oa":false,"landing_page_url":"https://doi.org/10.23919/acc53348.2022.9867291","pdf_url":null,"source":{"id":"https://openalex.org/S4363607732","display_name":"2022 American Control Conference (ACC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 American Control Conference (ACC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.7799999713897705,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":65,"referenced_works":["https://openalex.org/W314209163","https://openalex.org/W1922736325","https://openalex.org/W1971934985","https://openalex.org/W1996864164","https://openalex.org/W2028739770","https://openalex.org/W2042710419","https://openalex.org/W2043458914","https://openalex.org/W2071983464","https://openalex.org/W2097460613","https://openalex.org/W2099977591","https://openalex.org/W2104958104","https://openalex.org/W2119567691","https://openalex.org/W2139025031","https://openalex.org/W2153188087","https://openalex.org/W2155039147","https://openalex.org/W2601574318","https://openalex.org/W2650094097","https://openalex.org/W2726042758","https://openalex.org/W2783742282","https://openalex.org/W2889299993","https://openalex.org/W2914932468","https://openalex.org/W2919742679","https://openalex.org/W2949302905","https://openalex.org/W2949382134","https://openalex.org/W2963007855","https://openalex.org/W2963040514","https://openalex.org/W2963241635","https://openalex.org/W2963576501","https://openalex.org/W2964102336","https://openalex.org/W2964240094","https://openalex.org/W2975071282","https://openalex.org/W2997973415","https://openalex.org/W3008463948","https://openalex.org/W3011830603","https://openalex.org/W3013479538","https://openalex.org/W3016890209","https://openalex.org/W3035204268","https://openalex.org/W3037229067","https://openalex.org/W3042135496","https://openalex.org/W3105020018","https://openalex.org/W3164350488","https://openalex.org/W3169476548","https://openalex.org/W3184741403","https://openalex.org/W3211768774","https://openalex.org/W4206342194","https://openalex.org/W4214717370","https://openalex.org/W4288602110","https://openalex.org/W4297924967","https://openalex.org/W4298064558","https://openalex.org/W4307347247","https://openalex.org/W4391410851","https://openalex.org/W6640345655","https://openalex.org/W6739819759","https://openalex.org/W6751368208","https://openalex.org/W6751658861","https://openalex.org/W6758937863","https://openalex.org/W6759163136","https://openalex.org/W6763578770","https://openalex.org/W6767092952","https://openalex.org/W6771753067","https://openalex.org/W6780179245","https://openalex.org/W6785894376","https://openalex.org/W6795230381","https://openalex.org/W6796892462","https://openalex.org/W6803871076"],"related_works":["https://openalex.org/W2152670157","https://openalex.org/W2386410636","https://openalex.org/W176737593","https://openalex.org/W2808418668","https://openalex.org/W2903299703","https://openalex.org/W2016648086","https://openalex.org/W3105579180","https://openalex.org/W2156021013","https://openalex.org/W4399157305","https://openalex.org/W4281791088"],"abstract_inverted_index":{"Value-based":[0],"methods":[1,27,52],"play":[2],"a":[3,20,103],"fundamental":[4],"role":[5],"in":[6,64,83,102],"Markov":[7],"decision":[8],"processes":[9],"(MDPs)":[10],"and":[11,36,53,94,116,136],"reinforcement":[12],"learning":[13,40],"(RL).":[14],"In":[15],"this":[16],"paper,":[17],"we":[18,56],"present":[19],"unified":[21],"control-theoretic":[22],"framework":[23],"for":[24,72],"analyzing":[25],"valued-based":[26],"such":[28,124],"as":[29],"value":[30,33],"computation":[31],"(VC),":[32],"iteration":[34],"(VI),":[35],"temporal":[37],"difference":[38],"(TD)":[39],"(with":[41],"linear":[42,87],"function":[43],"approximation).":[44],"Built":[45],"upon":[46],"an":[47],"intrinsic":[48],"connection":[49],"between":[50,112],"value-based":[51,75],"dynamic":[54],"systems,":[55],"can":[57,95,126],"directly":[58],"use":[59],"existing":[60],"convex":[61,81],"testing":[62,78],"conditions":[63,79],"control":[65,114],"theory":[66,135],"to":[67,98],"derive":[68],"various":[69],"convergence":[70],"results":[71],"the":[73,131],"aforementioned":[74],"methods.":[76],"These":[77],"are":[80],"programs":[82],"form":[84],"of":[85,133],"either":[86],"programming":[88,92],"(LP)":[89],"or":[90],"semidefinite":[91],"(SDP),":[93],"be":[96],"solved":[97],"construct":[99],"Lyapunov":[100],"functions":[101],"straightforward":[104],"manner.":[105],"Our":[106],"analysis":[107],"reveals":[108],"some":[109],"intriguing":[110],"connections":[111,125],"feedback":[113],"systems":[115],"RL":[117],"algorithms.":[118],"It":[119],"is":[120],"our":[121],"hope":[122],"that":[123],"inspire":[127],"more":[128],"work":[129],"at":[130],"intersection":[132],"system/control":[134],"RL.":[137]},"counts_by_year":[{"year":2023,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}