{"id":"https://openalex.org/W3160601834","doi":"https://doi.org/10.1137/21m1421726","title":"A Stochastic Composite Augmented Lagrangian Method for Reinforcement Learning","display_name":"A Stochastic Composite Augmented Lagrangian Method for Reinforcement Learning","publication_year":2023,"publication_date":"2023-06-22","ids":{"openalex":"https://openalex.org/W3160601834","doi":"https://doi.org/10.1137/21m1421726","mag":"3160601834"},"language":"en","primary_location":{"id":"doi:10.1137/21m1421726","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1137/21m1421726","pdf_url":null,"source":{"id":"https://openalex.org/S928796702","display_name":"SIAM Journal on Optimization","issn_l":"1052-6234","issn":["1052-6234","1095-7189"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320508","host_organization_name":"Society for Industrial and Applied Mathematics","host_organization_lineage":["https://openalex.org/P4310320508"],"host_organization_lineage_names":["Society for Industrial and Applied Mathematics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SIAM Journal on Optimization","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2105.09716","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100371413","display_name":"Yongfeng Li","orcid":"https://orcid.org/0000-0001-5909-7546"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210133846","display_name":"Peking University International Hospital","ror":"https://ror.org/03jxhcr96","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210133846"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yongfeng Li","raw_affiliation_strings":["Beijing International Center for Mathematical Research, Peking University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing International Center for Mathematical Research, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570","https://openalex.org/I4210133846"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060494556","display_name":"Mingming Zhao","orcid":"https://orcid.org/0000-0002-6405-4652"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210133846","display_name":"Peking University International Hospital","ror":"https://ror.org/03jxhcr96","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210133846"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingming Zhao","raw_affiliation_strings":["Beijing International Center for Mathematical Research, Peking University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing International Center for Mathematical Research, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570","https://openalex.org/I4210133846"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100686054","display_name":"Weijie Chen","orcid":"https://orcid.org/0000-0003-3657-2943"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weijie Chen","raw_affiliation_strings":["Academy for Advanced Interdisciplinary Studies, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3657-2943","affiliations":[{"raw_affiliation_string":"Academy for Advanced Interdisciplinary Studies, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5006127137","display_name":"Zaiwen Wen","orcid":"https://orcid.org/0000-0003-1762-0671"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zaiwen Wen","raw_affiliation_strings":["Corresponding author. Beijing International Center for Mathematical Research, College of Engineering and Center for Machine Learning Research, Peking University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Corresponding author. Beijing International Center for Mathematical Research, College of Engineering and Center for Machine Learning Research, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5100371413"],"corresponding_institution_ids":["https://openalex.org/I20231570","https://openalex.org/I4210133846"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.00142182,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":"33","issue":"2","first_page":"921","last_page":"949"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11206","display_name":"Model Reduction and Neural Networks","score":0.9908999800682068,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/augmented-lagrangian-method","display_name":"Augmented Lagrangian method","score":0.8673855066299438},{"id":"https://openalex.org/keywords/lagrangian-relaxation","display_name":"Lagrangian relaxation","score":0.7233136892318726},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.659628689289093},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.6225133538246155},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.6083166599273682},{"id":"https://openalex.org/keywords/lagrange-multiplier","display_name":"Lagrange multiplier","score":0.5002009868621826},{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.4604000151157379},{"id":"https://openalex.org/keywords/quadratic-programming","display_name":"Quadratic programming","score":0.46026164293289185},{"id":"https://openalex.org/keywords/penalty-method","display_name":"Penalty method","score":0.4222186803817749},{"id":"https://openalex.org/keywords/quadratic-equation","display_name":"Quadratic equation","score":0.42051059007644653},{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.41557779908180237},{"id":"https://openalex.org/keywords/sequential-quadratic-programming","display_name":"Sequential quadratic programming","score":0.41290000081062317},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.31516018509864807},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.2522541582584381},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.16062626242637634}],"concepts":[{"id":"https://openalex.org/C150452318","wikidata":"https://www.wikidata.org/wiki/Q4820432","display_name":"Augmented Lagrangian method","level":2,"score":0.8673855066299438},{"id":"https://openalex.org/C91765299","wikidata":"https://www.wikidata.org/wiki/Q3424292","display_name":"Lagrangian relaxation","level":2,"score":0.7233136892318726},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.659628689289093},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.6225133538246155},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.6083166599273682},{"id":"https://openalex.org/C73684929","wikidata":"https://www.wikidata.org/wiki/Q598870","display_name":"Lagrange multiplier","level":2,"score":0.5002009868621826},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.4604000151157379},{"id":"https://openalex.org/C81845259","wikidata":"https://www.wikidata.org/wiki/Q290117","display_name":"Quadratic programming","level":2,"score":0.46026164293289185},{"id":"https://openalex.org/C6180225","wikidata":"https://www.wikidata.org/wiki/Q3411771","display_name":"Penalty method","level":2,"score":0.4222186803817749},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.42051059007644653},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.41557779908180237},{"id":"https://openalex.org/C198927703","wikidata":"https://www.wikidata.org/wiki/Q4373881","display_name":"Sequential quadratic programming","level":3,"score":0.41290000081062317},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.31516018509864807},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2522541582584381},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.16062626242637634},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1137/21m1421726","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1137/21m1421726","pdf_url":null,"source":{"id":"https://openalex.org/S928796702","display_name":"SIAM Journal on Optimization","issn_l":"1052-6234","issn":["1052-6234","1095-7189"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320508","host_organization_name":"Society for Industrial and Applied Mathematics","host_organization_lineage":["https://openalex.org/P4310320508"],"host_organization_lineage_names":["Society for Industrial and Applied Mathematics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SIAM Journal on Optimization","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2105.09716","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2105.09716","pdf_url":"https://arxiv.org/pdf/2105.09716","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},{"id":"doi:10.48550/arxiv.2105.09716","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2105.09716","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2105.09716","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2105.09716","pdf_url":"https://arxiv.org/pdf/2105.09716","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.7799999713897705,"display_name":"Peace, Justice and strong institutions"}],"awards":[{"id":"https://openalex.org/G2030148605","display_name":null,"funder_award_id":"11831002","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":40,"referenced_works":["https://openalex.org/W205960364","https://openalex.org/W1494085563","https://openalex.org/W1522301498","https://openalex.org/W1583089965","https://openalex.org/W1583330603","https://openalex.org/W1771410628","https://openalex.org/W2038497950","https://openalex.org/W2098432798","https://openalex.org/W2119567691","https://openalex.org/W2121863487","https://openalex.org/W2135779729","https://openalex.org/W2145339207","https://openalex.org/W2150468603","https://openalex.org/W2158782408","https://openalex.org/W2173248099","https://openalex.org/W2736601468","https://openalex.org/W2754517384","https://openalex.org/W2761873684","https://openalex.org/W2765415241","https://openalex.org/W2771966232","https://openalex.org/W2776124380","https://openalex.org/W2787938642","https://openalex.org/W2806985155","https://openalex.org/W2949608212","https://openalex.org/W2949996623","https://openalex.org/W2956068307","https://openalex.org/W2963218690","https://openalex.org/W2963864421","https://openalex.org/W2964291307","https://openalex.org/W2980326798","https://openalex.org/W3035600116","https://openalex.org/W3046149626","https://openalex.org/W3099050578","https://openalex.org/W3103780890","https://openalex.org/W3117137507","https://openalex.org/W4214717370","https://openalex.org/W4229650096","https://openalex.org/W4287759271","https://openalex.org/W4298023569","https://openalex.org/W4394666657"],"related_works":["https://openalex.org/W2015060724","https://openalex.org/W2002102264","https://openalex.org/W4237964977","https://openalex.org/W2058578573","https://openalex.org/W2360271985","https://openalex.org/W3207854495","https://openalex.org/W1968991653","https://openalex.org/W2030716009","https://openalex.org/W2053344917","https://openalex.org/W2017965889"],"abstract_inverted_index":{"In":[0],"this":[1],"paper,":[2],"we":[3,82],"consider":[4],"the":[5,17,21,30,44,49,52,57,61,65,77,80,84,88,94,99,111,119,123,138,145,150,154,157,166,176,183],"linear":[6],"programming":[7],"(LP)":[8],"formulation":[9],"for":[10],"deep":[11,103],"reinforcement":[12],"learning.":[13],"The":[14,38],"number":[15],"of":[16,23,79,144,153],"constraints":[18],"depends":[19],"on":[20,165],"size":[22],"state":[24],"and":[25,60,73,187],"action":[26],"spaces,":[27],"which":[28],"makes":[29],"problem":[31],"intractable":[32,95],"in":[33,47,64,71,86,122,185],"large":[34],"or":[35],"continuous":[36],"environments.":[37],"general":[39,133],"augmented":[40,66,89,124],"Lagrangian":[41,67,90,106,125],"method":[42,107,126,198],"suffers":[43],"double-sampling":[45],"obstacle":[46],"solving":[48],"LP.":[50],"Namely,":[51],"conditional":[53,96],"expectations":[54,97],"originated":[55],"from":[56,76,141],"constraint":[58],"functions":[59],"quadratic":[62,167],"penalties":[63],"function":[68,91],"impose":[69],"difficulties":[70],"sampling":[72],"evaluation.":[74],"Motivated":[75],"updates":[78],"multipliers,":[81],"overcome":[83],"obstacles":[85],"minimizing":[87],"by":[92],"replacing":[93],"with":[98],"multipliers.":[100],"Therefore,":[101],"a":[102,114,128,142],"parameterized":[104],"augment":[105],"is":[108,159,190,199],"proposed.":[109],"Furthermore,":[110],"replacement":[112],"provides":[113],"promising":[115],"breakthrough":[116],"to":[117,149,201],"integrate":[118],"two":[120],"steps":[121],"into":[127],"single":[129],"constrained":[130,146],"problem.":[131],"A":[132,162],"theoretical":[134,163],"analysis":[135,164],"shows":[136,175],"that":[137,196],"solutions":[139],"generated":[140],"sequence":[143],"optimizations":[147],"converge":[148],"optimal":[151],"solution":[152],"LP":[155],"if":[156,182],"error":[158],"controlled":[160],"properly.":[161],"penalty":[168],"algorithm":[169,189],"under":[170],"neural":[171],"tangent":[172],"kernel":[173],"setting":[174],"residual":[177],"can":[178],"be":[179],"arbitrarily":[180],"small":[181],"parameter":[184],"network":[186],"optimization":[188],"chosen":[191],"suitably.":[192],"Preliminary":[193],"experiments":[194],"illustrate":[195],"our":[197],"competitive":[200],"other":[202],"state-of-the-art":[203],"algorithms.":[204]},"counts_by_year":[{"year":2022,"cited_by_count":1}],"updated_date":"2025-12-19T19:40:27.379048","created_date":"2021-05-24T00:00:00"}