{"id":"https://openalex.org/W2997273686","doi":"https://doi.org/10.48550/arxiv.2008.11811","title":"Constrained Markov Decision Processes via Backward Value Functions","display_name":"Constrained Markov Decision Processes via Backward Value Functions","publication_year":2020,"publication_date":"2020-08-26","ids":{"openalex":"https://openalex.org/W2997273686","doi":"https://doi.org/10.48550/arxiv.2008.11811","mag":"2997273686"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2008.11811","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2008.11811","pdf_url":"https://arxiv.org/pdf/2008.11811","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2008.11811","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055937250","display_name":"Harsh Satija","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Satija, Harsh","raw_affiliation_strings":["McGill University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"McGill University","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051361569","display_name":"Philip Amortila","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Amortila, Philip","raw_affiliation_strings":["McGill University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"McGill University","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5080591144","display_name":"Jo\u00eblle Pineau","orcid":"https://orcid.org/0000-0003-0747-7250","observed_orcids":["https://orcid.org/0000-0003-0747-7250"]},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Pineau, Joelle","raw_affiliation_strings":["McGill University / Facebook"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"McGill University / Facebook","institution_ids":["https://openalex.org/I5023651"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I5023651"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":30,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"1","issue":null,"first_page":"8502","last_page":"8511"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.0,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/constrained-markov-decision-process","display_name":"constrained Markov decision process","score":0.9959999918937683},{"id":"https://openalex.org/keywords/on-policy-learning","display_name":"on-policy learning","score":0.6029999852180481},{"id":"https://openalex.org/keywords/navigation-safety","display_name":"navigation safety","score":0.7279999852180481},{"id":"https://openalex.org/keywords/convergence-guarantees","display_name":"convergence guarantees","score":0.6909999847412109}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8713439702987671},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.8684669733047485},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7229300737380981},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.5996249914169312},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5976239442825317},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5658334493637085},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.5148541331291199},{"id":"https://openalex.org/C14646407","wikidata":"https://www.wikidata.org/wiki/Q1430750","display_name":"Bellman equation","level":2,"score":0.4965105652809143},{"id":"https://openalex.org/C188116033","wikidata":"https://www.wikidata.org/wiki/Q2664563","display_name":"Q-learning","level":3,"score":0.47019442915916443},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4170989692211151},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11974161863327026},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"pmh:oai:arXiv.org:2008.11811","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2008.11811","pdf_url":"https://arxiv.org/pdf/2008.11811","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3080734044","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2008.11811","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2008.11811","doi":"https://doi.org/10.48550/arxiv.2008.11811","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2008.11811","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"mag:2997273686","doi":null,"is_oa":false,"landing_page_url":"https://openreview.net/pdf?id=S1lyyANYwr","pdf_url":null,"source":{"id":"https://openalex.org/S4306419644","display_name":"International Conference on Machine Learning","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"International Conference on Machine Learning","raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2008.11811","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2008.11811","pdf_url":"https://arxiv.org/pdf/2008.11811","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2997273686.pdf","grobid_xml":"https://content.openalex.org/works/W2997273686.grobid-xml"},"referenced_works_count":14,"referenced_works":["https://openalex.org/W1876490351","https://openalex.org/W2151237105","https://openalex.org/W2616964725","https://openalex.org/W2619268125","https://openalex.org/W2736601468","https://openalex.org/W2768908787","https://openalex.org/W2784465508","https://openalex.org/W2803543472","https://openalex.org/W2804791273","https://openalex.org/W2913300629","https://openalex.org/W2953228110","https://openalex.org/W2963780574","https://openalex.org/W3035642820","https://openalex.org/W3037207827"],"related_works":["https://openalex.org/W3130575077","https://openalex.org/W3214770989","https://openalex.org/W3189576409","https://openalex.org/W2978598593","https://openalex.org/W4396718754","https://openalex.org/W3046384803","https://openalex.org/W3189256393","https://openalex.org/W2803543472","https://openalex.org/W4407949724","https://openalex.org/W4378501646"],"abstract_inverted_index":{"Although":[0],"Reinforcement":[1],"Learning":[2],"(RL)":[3],"algorithms":[4],"have":[5],"found":[6],"tremendous":[7],"success":[8],"in":[9,22,54,68,178],"simulated":[10],"domains,":[11],"they":[12],"often":[13],"cannot":[14],"directly":[15],"be":[16],"applied":[17],"to":[18,29,43,110],"physical":[19],"systems,":[20],"especially":[21],"cases":[23],"where":[24],"there":[25],"are":[26,135],"hard":[27],"constraints":[28,87,114,134],"satisfy":[30],"(e.g.":[31],"on":[32,173],"safety":[33,151],"or":[34,65],"resources).":[35],"In":[36,77],"standard":[37],"RL,":[38],"the":[39,55,63,66,73,82,133,146,153,160],"agent":[40,67,147],"is":[41,109,171],"incentivized":[42],"explore":[44],"any":[45],"behavior":[46,59],"as":[47,49,88],"long":[48],"it":[50],"maximizes":[51,128],"rewards,":[52],"but":[53],"real":[56],"world,":[57],"undesired":[58],"can":[60],"damage":[61],"either":[62],"system":[64],"a":[69,89,96,122],"way":[70],"that":[71,132],"breaks":[72],"learning":[74,85],"process":[75],"itself.":[76],"this":[78,164],"work,":[79],"we":[80,120],"model":[81],"problem":[83],"of":[84,106,155,163,168,181],"with":[86,184],"Constrained":[90],"Markov":[91],"Decision":[92],"Process":[93],"and":[94,177],"provide":[95,141],"new":[97],"on-policy":[98],"formulation":[99],"for":[100],"solving":[101],"it.":[102],"A":[103],"key":[104],"contribution":[105],"our":[107,169],"approach":[108,170],"translate":[111],"cumulative":[112],"cost":[113],"into":[115],"state-based":[116],"constraints.":[117],"Through":[118],"this,":[119],"define":[121],"safe":[123,174],"policy":[124],"improvement":[125],"method":[126],"which":[127,145],"returns":[129],"while":[130,149],"ensuring":[131,150],"satisfied":[136],"at":[137],"every":[138],"step.":[139],"We":[140,157],"theoretical":[142],"guarantees":[143],"under":[144],"converges":[148],"over":[152],"course":[154],"training.":[156],"also":[158],"highlight":[159],"computational":[161],"advantages":[162],"approach.":[165],"The":[166],"effectiveness":[167],"demonstrated":[172],"navigation":[175],"tasks":[176],"safety-constrained":[179],"versions":[180],"MuJoCo":[182],"environments,":[183],"deep":[185],"neural":[186],"networks.":[187]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":2}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2020-01-10T00:00:00"}