{"id":"https://openalex.org/W3214770989","doi":null,"title":"Safe Exploration for Constrained Reinforcement Learning with Provable Guarantees.","display_name":"Safe Exploration for Constrained Reinforcement Learning with Provable Guarantees.","publication_year":2021,"publication_date":"2021-12-01","ids":{"openalex":"https://openalex.org/W3214770989","mag":"3214770989"},"language":"en","primary_location":{"id":"mag:3214770989","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/pdf/2112.00885.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/pdf/2112.00885.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005327552","display_name":"Archana Bura","orcid":"https://orcid.org/0000-0001-7897-2473","observed_orcids":["https://orcid.org/0000-0001-7897-2473"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Archana Bura","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-7897-2473","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056952829","display_name":"Aria HasanzadeZonuzy","orcid":"https://orcid.org/0000-0002-9701-1405","observed_orcids":["https://orcid.org/0000-0002-9701-1405"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aria HasanzadeZonuzy","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-9701-1405","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053096993","display_name":"Dileep M. Kalathil","orcid":"https://orcid.org/0000-0001-7968-5185","observed_orcids":["https://orcid.org/0000-0001-7968-5185","https://orcid.org/0000-0002-7403-4006"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dileep Kalathil","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-7968-5185","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005504863","display_name":"Srinivas Shakkottai","orcid":"https://orcid.org/0000-0002-5882-6433","observed_orcids":["https://orcid.org/0000-0002-5882-6433"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Srinivas Shakkottai","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-5882-6433","affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5069801730","display_name":"Jean\u2010Fran\u00e7ois Chamberland","orcid":"https://orcid.org/0000-0002-2983-9884","observed_orcids":["https://orcid.org/0000-0002-2983-9884"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jean\u2010Fran\u00e7ois Chamberland","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-2983-9884","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.0,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/safe-reinforcement-learning","display_name":"safe reinforcement learning","score":0.9340000152587891},{"id":"https://openalex.org/keywords/constrained-markov-decision-process","display_name":"constrained Markov decision process","score":0.8190000057220459},{"id":"https://openalex.org/keywords/model-based-reinforcement-learning","display_name":"model-based reinforcement learning","score":0.7509999871253967},{"id":"https://openalex.org/keywords/regret-bounds","display_name":"regret bounds","score":0.6240000128746033},{"id":"https://openalex.org/keywords/cumulative-regret","display_name":"cumulative regret","score":0.6050000190734863},{"id":"https://openalex.org/keywords/safety-gaps","display_name":"safety gaps","score":0.6200000047683716},{"id":"https://openalex.org/keywords/unknown-transition-probabilities","display_name":"unknown transition probabilities","score":0.6729999780654907}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8057353496551514},{"id":"https://openalex.org/C50817715","wikidata":"https://www.wikidata.org/wiki/Q79895177","display_name":"Regret","level":2,"score":0.7210930585861206},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.6779729127883911},{"id":"https://openalex.org/C14646407","wikidata":"https://www.wikidata.org/wiki/Q1430750","display_name":"Bellman equation","level":2,"score":0.5218802094459534},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5001134872436523},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4671195447444916},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4003687798976898},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36048927903175354},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3433445394039154},{"id":"https://openalex.org/C144237770","wikidata":"https://www.wikidata.org/wiki/Q747534","display_name":"Mathematical economics","level":1,"score":0.33016952872276306},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.2505168616771698},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.18650966882705688},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:3214770989","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/pdf/2112.00885.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:3214770989","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/pdf/2112.00885.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W1061340565","https://openalex.org/W1518931405","https://openalex.org/W1786332878","https://openalex.org/W1850488217","https://openalex.org/W1915973093","https://openalex.org/W1992168592","https://openalex.org/W2012136789","https://openalex.org/W2070570138","https://openalex.org/W2073314543","https://openalex.org/W2098432798","https://openalex.org/W2150234726","https://openalex.org/W2415697701","https://openalex.org/W2427917354","https://openalex.org/W2513180554","https://openalex.org/W2552231430","https://openalex.org/W2592542720","https://openalex.org/W2618318883","https://openalex.org/W2749843017","https://openalex.org/W2788084076","https://openalex.org/W2804791273","https://openalex.org/W2914526782","https://openalex.org/W2921637089","https://openalex.org/W2946061571","https://openalex.org/W2947223001","https://openalex.org/W2962803570","https://openalex.org/W2963082979","https://openalex.org/W2963169929","https://openalex.org/W2963195786","https://openalex.org/W2963582321","https://openalex.org/W2964054583","https://openalex.org/W2966735560","https://openalex.org/W2966956172","https://openalex.org/W2990389059","https://openalex.org/W2990466689","https://openalex.org/W2994712737","https://openalex.org/W2997273133","https://openalex.org/W3008953696","https://openalex.org/W3009922106","https://openalex.org/W3034840734","https://openalex.org/W3035759338","https://openalex.org/W3046384803","https://openalex.org/W3087810330","https://openalex.org/W3101215519","https://openalex.org/W3101517963","https://openalex.org/W3102182290","https://openalex.org/W3102309758","https://openalex.org/W3106002372","https://openalex.org/W3158382280","https://openalex.org/W3212552393","https://openalex.org/W4327630357"],"related_works":["https://openalex.org/W3171019181","https://openalex.org/W7133226125","https://openalex.org/W3211584761","https://openalex.org/W7133219967","https://openalex.org/W3189256393","https://openalex.org/W4320342431","https://openalex.org/W3180716444","https://openalex.org/W4407949724","https://openalex.org/W4391667216","https://openalex.org/W4392939469"],"abstract_inverted_index":{"We":[0,27,75,238],"consider":[1],"the":[2,37,56,62,85,107,115,121,127,132,142,146,149,153,166,169,211,217],"problem":[3,35],"of":[4,39,73,117,123,134,155,190,219,235],"learning":[5,24,33,135,210],"an":[6,13,48,97,201],"episodic":[7],"safe":[8,79,158],"control":[9],"policy":[10],"that":[11,66,82,94,181,229],"minimizes":[12],"objective":[14],"function,":[15],"while":[16,222],"satisfying":[17],"necessary":[18],"safety":[19,30,57,108,143,193,236],"constraints":[20,60,109,173],"--":[21],"both":[22],"during":[23,70,110,177],"and":[25,92,137,152],"deployment.":[26],"formulate":[28],"this":[29],"constrained":[31],"reinforcement":[32],"(RL)":[34],"using":[36],"framework":[38],"a":[40,77,156,192,224],"finite-horizon":[41],"Constrained":[42],"Markov":[43],"Decision":[44],"Process":[45],"(CMDP)":[46],"with":[47,205],"unknown":[49,220],"transition":[50],"probability":[51],"function.":[52],"Here,":[53],"we":[54,83],"model":[55],"requirements":[58],"as":[59,163,168],"on":[61,246],"expected":[63],"cumulative":[64,103],"costs":[65],"must":[67],"be":[68,175],"satisfied":[69],"all":[71],"episodes":[72],"learning.":[74],"propose":[76],"model-based":[78],"RL":[80],"algorithm":[81,183,241],"call":[84],"Optimistic-Pessimistic":[86],"Safe":[87],"Reinforcement":[88],"Learning":[89],"(OPSRL)":[90],"algorithm,":[91],"show":[93],"it":[95],"achieves":[96],"$\\tilde{\\mathcal{O}}(S^{2}\\sqrt{A":[98],"H^{7}K}/":[99],"(\\bar{C}":[100],"-":[101,139],"\\bar{C}_{b}))$":[102],"regret":[104,187],"without":[105],"violating":[106],"learning,":[111,178],"where":[112,172],"$S$":[113],"is":[114,120,126,131,141,165,198],"number":[116,122,133],"states,":[118],"$A$":[119],"actions,":[124],"$H$":[125],"horizon":[128],"length,":[129],"$K$":[130],"episodes,":[136],"$(\\bar{C}":[138],"\\bar{C}_{b})$":[140],"gap,":[144],"i.e.,":[145],"difference":[147],"between":[148],"constraint":[150,207],"value":[151],"cost":[154],"known":[157],"baseline":[159],"policy.":[160,212],"The":[161],"scaling":[162],"$\\tilde{\\mathcal{O}}(\\sqrt{K})$":[164],"same":[167],"traditional":[170],"approach":[171,204,214],"may":[174],"violated":[176],"which":[179],"means":[180],"our":[182,240],"suffers":[184],"no":[185],"additional":[186],"in":[188],"spite":[189],"providing":[191],"guarantee.":[194],"Our":[195],"key":[196],"idea":[197],"to":[199,232],"use":[200],"optimistic":[202],"exploration":[203,218],"pessimistic":[206],"enforcement":[208],"for":[209,226],"This":[213],"simultaneously":[215],"incentivizes":[216],"states":[221,228],"imposing":[223],"penalty":[225],"visiting":[227],"are":[230],"likely":[231],"cause":[233],"violation":[234],"constraints.":[237],"validate":[239],"by":[242],"evaluating":[243],"its":[244],"performance":[245],"benchmark":[247],"problems":[248],"against":[249],"conventional":[250],"approaches.":[251]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2021-12-06T00:00:00"}