{"id":"https://openalex.org/W3189576409","doi":"https://doi.org/10.24963/ijcai.2021/614","title":"Policy Learning with Constraints in Model-free Reinforcement Learning: A Survey","display_name":"Policy Learning with Constraints in Model-free Reinforcement Learning: A Survey","publication_year":2021,"publication_date":"2021-08-01","ids":{"openalex":"https://openalex.org/W3189576409","doi":"https://doi.org/10.24963/ijcai.2021/614","mag":"3189576409"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2021/614","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2021/614","pdf_url":"https://www.ijcai.org/proceedings/2021/0614.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirtieth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.ijcai.org/proceedings/2021/0614.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065806048","display_name":"Yongshuai Liu","orcid":"https://orcid.org/0000-0002-9629-0393"},"institutions":[{"id":"https://openalex.org/I84218800","display_name":"University of California, Davis","ror":"https://ror.org/05rrcem69","country_code":"US","type":"education","lineage":["https://openalex.org/I84218800"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Yongshuai Liu","raw_affiliation_strings":["University of California, Davis"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California, Davis","institution_ids":["https://openalex.org/I84218800"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048314614","display_name":"Avishai Halev","orcid":"https://orcid.org/0009-0004-4348-5447"},"institutions":[{"id":"https://openalex.org/I4210105028","display_name":"Total (United States)","ror":"https://ror.org/01ajr8702","country_code":"US","type":"company","lineage":["https://openalex.org/I103084370","https://openalex.org/I4210105028"]},{"id":"https://openalex.org/I84218800","display_name":"University of California, Davis","ror":"https://ror.org/05rrcem69","country_code":"US","type":"education","lineage":["https://openalex.org/I84218800"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Avishai Halev","raw_affiliation_strings":["Total E&P R&T USA","University of California, Davis"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Total E&P R&T USA","institution_ids":["https://openalex.org/I4210105028"]},{"raw_affiliation_string":"University of California, Davis","institution_ids":["https://openalex.org/I84218800"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100352241","display_name":"Xin Liu","orcid":"https://orcid.org/0000-0002-5379-8269"},"institutions":[{"id":"https://openalex.org/I84218800","display_name":"University of California, Davis","ror":"https://ror.org/05rrcem69","country_code":"US","type":"education","lineage":["https://openalex.org/I84218800"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xin Liu","raw_affiliation_strings":["University of California, Davis"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California, Davis","institution_ids":["https://openalex.org/I84218800"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5065806048"],"corresponding_institution_ids":["https://openalex.org/I84218800"],"apc_list":null,"apc_paid":null,"fwci":8.1159,"has_fulltext":false,"cited_by_count":83,"citation_normalized_percentile":{"value":0.97938908,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"4508","last_page":"4515"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9585999846458435,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9585999846458435,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.8938742876052856},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.75909024477005},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.7400965690612793},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5988360047340393},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5690326690673828},{"id":"https://openalex.org/keywords/limit","display_name":"Limit (mathematics)","score":0.4646574556827545},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.43186986446380615},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4286418557167053},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4094179570674896},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08017703890800476}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8938742876052856},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.75909024477005},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.7400965690612793},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5988360047340393},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5690326690673828},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.4646574556827545},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.43186986446380615},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4286418557167053},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4094179570674896},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08017703890800476},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.24963/ijcai.2021/614","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2021/614","pdf_url":"https://www.ijcai.org/proceedings/2021/0614.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirtieth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.24963/ijcai.2021/614","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2021/614","pdf_url":"https://www.ijcai.org/proceedings/2021/0614.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirtieth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.5799999833106995}],"awards":[{"id":"https://openalex.org/G1237098786","display_name":null,"funder_award_id":"CNS-1718901","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2573420999","display_name":"NSF Convergence Accelerator - Track D: Data-Driven Disease Control and Prevention in Veterinary Health","funder_award_id":"2040680","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3441695903","display_name":null,"funder_award_id":"020-67021-32855","funder_id":"https://openalex.org/F4320306114","funder_display_name":"U.S. Department of Agriculture"},{"id":"https://openalex.org/G3789803274","display_name":"BIGDATA: IA: A multi-level approach for global optimization of the surveillance and control of infectious disease in the swine industry","funder_award_id":"1838207","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3841099640","display_name":"CNS Core: Medium: Collaborative:  Exploring and Exploiting Learning for Efficient Network Control: Non-Stationarity,  Inter-Dependence, and Domain-Knowledge","funder_award_id":"1901218","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4214077137","display_name":null,"funder_award_id":"IIS-1838207","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5173218366","display_name":null,"funder_award_id":"USDA-020-67021-32855","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5870508795","display_name":"NeTS: Small: Learning-Guided Network Resource Allocation: A Closed-Loop Approach","funder_award_id":"1718901","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G816430435","display_name":null,"funder_award_id":"CNS 1901218","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306114","display_name":"U.S. Department of Agriculture","ror":"https://ror.org/01na82s61"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3189576409.pdf","grobid_xml":"https://content.openalex.org/works/W3189576409.grobid-xml"},"referenced_works_count":46,"referenced_works":["https://openalex.org/W41554520","https://openalex.org/W1506085041","https://openalex.org/W1518931405","https://openalex.org/W1702271787","https://openalex.org/W1771410628","https://openalex.org/W1967459934","https://openalex.org/W2107726111","https://openalex.org/W2138108551","https://openalex.org/W2155007355","https://openalex.org/W2155027007","https://openalex.org/W2401992734","https://openalex.org/W2427917354","https://openalex.org/W2736601468","https://openalex.org/W2736629007","https://openalex.org/W2740828027","https://openalex.org/W2784465508","https://openalex.org/W2788014517","https://openalex.org/W2788084076","https://openalex.org/W2789084548","https://openalex.org/W2804791273","https://openalex.org/W2911618937","https://openalex.org/W2913300629","https://openalex.org/W2962734844","https://openalex.org/W2963049774","https://openalex.org/W2963293747","https://openalex.org/W2979211489","https://openalex.org/W2982316857","https://openalex.org/W2994712737","https://openalex.org/W2998619042","https://openalex.org/W3007236068","https://openalex.org/W3034840734","https://openalex.org/W3080734044","https://openalex.org/W3103767196","https://openalex.org/W3110506085","https://openalex.org/W3162902207","https://openalex.org/W3179181747","https://openalex.org/W4287368046","https://openalex.org/W4287725923","https://openalex.org/W4287829052","https://openalex.org/W4287863949","https://openalex.org/W4288363736","https://openalex.org/W4293545785","https://openalex.org/W4294183581","https://openalex.org/W4294562617","https://openalex.org/W4297336573","https://openalex.org/W4298857966"],"related_works":["https://openalex.org/W3096874164","https://openalex.org/W2937181779","https://openalex.org/W2386410636","https://openalex.org/W1985560493","https://openalex.org/W2357975469","https://openalex.org/W2145363145","https://openalex.org/W1626977535","https://openalex.org/W4284974072","https://openalex.org/W2341346307","https://openalex.org/W4225269853"],"abstract_inverted_index":{"Reinforcement":[0],"Learning":[1],"(RL)":[2],"algorithms":[3],"have":[4],"had":[5],"tremendous":[6],"success":[7],"in":[8,23,57,67,84],"simulated":[9],"domains.":[10],"These":[11],"algorithms,":[12],"however,":[13,61],"often":[14],"cannot":[15],"be":[16],"directly":[17],"applied":[18],"to":[19,29,32,46],"physical":[20],"systems,":[21],"especially":[22],"cases":[24],"where":[25],"there":[26],"are":[27],"constraints":[28,66,83,95],"satisfy":[30],"(e.g.":[31],"ensure":[33],"safety":[34],"or":[35],"limit":[36],"resource":[37],"consumption).":[38],"In":[39,75],"standard":[40,133],"RL,":[41],"the":[42,51,58,68,90],"agent":[43],"is":[44,70],"incentivized":[45],"explore":[47],"any":[48],"policy":[49,124],"with":[50,94],"sole":[52],"goal":[53],"of":[54,64,92,107,132,141],"maximizing":[55],"reward;":[56],"real":[59],"world,":[60],"ensuring":[62],"satisfaction":[63],"certain":[65],"process":[69],"also":[71,138],"necessary":[72],"and":[73,102,110,116,120,135,144],"essential.":[74],"this":[76],"article,":[77],"we":[78,128],"overview":[79],"existing":[80,114],"approaches":[81,115],"addressing":[82],"model-free":[85],"reinforcement":[86],"learning.":[87],"We":[88,112,137],"model":[89],"problem":[91],"learning":[93],"as":[96],"a":[97,130],"Constrained":[98],"Markov":[99],"Decision":[100],"Process":[101],"consider":[103],"two":[104],"main":[105],"types":[106],"constraints:":[108],"cumulative":[109],"instantaneous.":[111],"summarize":[113,139],"discuss":[117],"their":[118],"pros":[119],"cons.":[121],"To":[122],"evaluate":[123],"performance":[125],"under":[126],"constraints,":[127],"introduce":[129],"set":[131],"benchmarks":[134],"metrics.":[136],"limitations":[140],"current":[142],"methods":[143],"present":[145],"open":[146],"questions":[147],"for":[148],"future":[149],"research.":[150]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":21},{"year":2024,"cited_by_count":19},{"year":2023,"cited_by_count":30},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":2}],"updated_date":"2026-05-10T08:33:47.465468","created_date":"2025-10-10T00:00:00"}