{"id":"https://openalex.org/W3183820536","doi":"https://doi.org/10.23919/acc50511.2021.9482783","title":"Beyond Cumulative Returns via Reinforcement Learning over State-Action Occupancy Measures","display_name":"Beyond Cumulative Returns via Reinforcement Learning over State-Action Occupancy Measures","publication_year":2021,"publication_date":"2021-05-25","ids":{"openalex":"https://openalex.org/W3183820536","doi":"https://doi.org/10.23919/acc50511.2021.9482783","mag":"3183820536"},"language":"en","primary_location":{"id":"doi:10.23919/acc50511.2021.9482783","is_oa":false,"landing_page_url":"https://doi.org/10.23919/acc50511.2021.9482783","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 American Control Conference (ACC)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101898224","display_name":"Junyu Zhang","orcid":"https://orcid.org/0000-0003-2194-9664"},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Junyu Zhang","raw_affiliation_strings":["Department of Operations Research and Financial Engineering, Princeton University, Princeton, NJ"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Operations Research and Financial Engineering, Princeton University, Princeton, NJ","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039563144","display_name":"Amrit Singh Bedi","orcid":"https://orcid.org/0000-0002-8807-2695"},"institutions":[{"id":"https://openalex.org/I166416128","display_name":"DEVCOM Army Research Laboratory","ror":"https://ror.org/011hc8f90","country_code":"US","type":"government","lineage":["https://openalex.org/I1304082316","https://openalex.org/I1330347796","https://openalex.org/I166416128","https://openalex.org/I2802705668","https://openalex.org/I4210154437"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amrit Singh Bedi","raw_affiliation_strings":["CISD, US Army Research Laboratory, Adelphi, MD, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CISD, US Army Research Laboratory, Adelphi, MD, USA","institution_ids":["https://openalex.org/I166416128"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100707460","display_name":"Mengdi Wang","orcid":"https://orcid.org/0000-0002-2101-9507"},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mengdi Wang","raw_affiliation_strings":["Department of Operations Research and Financial Engineering, Princeton University, Princeton, NJ"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Operations Research and Financial Engineering, Princeton University, Princeton, NJ","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025896653","display_name":"Alec Koppel","orcid":"https://orcid.org/0000-0003-2447-2873"},"institutions":[{"id":"https://openalex.org/I166416128","display_name":"DEVCOM Army Research Laboratory","ror":"https://ror.org/011hc8f90","country_code":"US","type":"government","lineage":["https://openalex.org/I1304082316","https://openalex.org/I1330347796","https://openalex.org/I166416128","https://openalex.org/I2802705668","https://openalex.org/I4210154437"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alec Koppel","raw_affiliation_strings":["CISD, US Army Research Laboratory, Adelphi, MD, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CISD, US Army Research Laboratory, Adelphi, MD, USA","institution_ids":["https://openalex.org/I166416128"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.4396,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.61092798,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"894","last_page":"901"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10328","display_name":"Supply Chain and Inventory Management","score":0.9868000149726868,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11413","display_name":"Risk and Portfolio Optimization","score":0.9775999784469604,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7999436855316162},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.7466971278190613},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.5901812314987183},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.565860390663147},{"id":"https://openalex.org/keywords/risk-measure","display_name":"Risk measure","score":0.484607070684433},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.44641199707984924},{"id":"https://openalex.org/keywords/cardinality","display_name":"Cardinality (data modeling)","score":0.43521198630332947},{"id":"https://openalex.org/keywords/linear-programming","display_name":"Linear programming","score":0.4208138883113861},{"id":"https://openalex.org/keywords/q-learning","display_name":"Q-learning","score":0.420434832572937},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.3465419411659241},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.31640517711639404},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.14528438448905945},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.09768995642662048}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7999436855316162},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.7466971278190613},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.5901812314987183},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.565860390663147},{"id":"https://openalex.org/C2781472820","wikidata":"https://www.wikidata.org/wiki/Q2154759","display_name":"Risk measure","level":3,"score":0.484607070684433},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.44641199707984924},{"id":"https://openalex.org/C87117476","wikidata":"https://www.wikidata.org/wiki/Q362383","display_name":"Cardinality (data modeling)","level":2,"score":0.43521198630332947},{"id":"https://openalex.org/C41045048","wikidata":"https://www.wikidata.org/wiki/Q202843","display_name":"Linear programming","level":2,"score":0.4208138883113861},{"id":"https://openalex.org/C188116033","wikidata":"https://www.wikidata.org/wiki/Q2664563","display_name":"Q-learning","level":3,"score":0.420434832572937},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3465419411659241},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31640517711639404},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.14528438448905945},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.09768995642662048},{"id":"https://openalex.org/C106159729","wikidata":"https://www.wikidata.org/wiki/Q2294553","display_name":"Financial economics","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C2780821815","wikidata":"https://www.wikidata.org/wiki/Q5340806","display_name":"Portfolio","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.23919/acc50511.2021.9482783","is_oa":false,"landing_page_url":"https://doi.org/10.23919/acc50511.2021.9482783","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 American Control Conference (ACC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.800000011920929,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":49,"referenced_works":["https://openalex.org/W51049863","https://openalex.org/W299346747","https://openalex.org/W1191599655","https://openalex.org/W1529558080","https://openalex.org/W1557287189","https://openalex.org/W1745373831","https://openalex.org/W1757796397","https://openalex.org/W1969147614","https://openalex.org/W1972190631","https://openalex.org/W1992586302","https://openalex.org/W2001009060","https://openalex.org/W2019291268","https://openalex.org/W2099506495","https://openalex.org/W2121863487","https://openalex.org/W2126282494","https://openalex.org/W2128521145","https://openalex.org/W2170923204","https://openalex.org/W2236244207","https://openalex.org/W2462780152","https://openalex.org/W2562316362","https://openalex.org/W2765415241","https://openalex.org/W2765892966","https://openalex.org/W2798543980","https://openalex.org/W2951520685","https://openalex.org/W2962803570","https://openalex.org/W2963082979","https://openalex.org/W2963423916","https://openalex.org/W2970036354","https://openalex.org/W2970927156","https://openalex.org/W2981915592","https://openalex.org/W2983617607","https://openalex.org/W2998059334","https://openalex.org/W3007455034","https://openalex.org/W3011338904","https://openalex.org/W3123298421","https://openalex.org/W3124407081","https://openalex.org/W3160598148","https://openalex.org/W3162902207","https://openalex.org/W4214717370","https://openalex.org/W4293545785","https://openalex.org/W4298857966","https://openalex.org/W4298876402","https://openalex.org/W6683300800","https://openalex.org/W6685029077","https://openalex.org/W6687063787","https://openalex.org/W6737893269","https://openalex.org/W6767112054","https://openalex.org/W6767486501","https://openalex.org/W6807091881"],"related_works":["https://openalex.org/W2808418668","https://openalex.org/W2101748387","https://openalex.org/W2357975469","https://openalex.org/W4380550992","https://openalex.org/W4322760752","https://openalex.org/W3096874164","https://openalex.org/W2970347269","https://openalex.org/W2146763310","https://openalex.org/W3167472281","https://openalex.org/W2937181779"],"abstract_inverted_index":{"We":[0,122],"study":[1],"the":[2,36,64,68,79,90,125,142,145,156,160,163,168,175],"estimation":[3],"of":[4,52,67,73,82,89,110,127,135,144,159,162,170,177],"risk-sensitive":[5,39],"policies":[6],"in":[7,100,152],"reinforcement":[8,74],"learning":[9],"problems":[10],"defined":[11],"by":[12,31],"a":[13,49,59,83,87,107],"Markov":[14],"Decision":[15],"Process":[16],"(MDPs)":[17],"whose":[18],"state":[19,93,146],"and":[20,147],"action":[21,148],"spaces":[22],"are":[23,28,41],"countably":[24],"finite.":[25],"Prior":[26],"efforts":[27],"predominately":[29],"afflicted":[30],"computational":[32,182],"challenges":[33],"associated":[34],"with":[35],"fact":[37],"that":[38,113,124],"MDPs":[40],"time-inconsistent.":[42],"To":[43,96],"ameliorate":[44],"this":[45,98,136,171],"issue,":[46],"we":[47,55,105],"propose":[48,106],"new":[50],"definition":[51],"risk,":[53],"which":[54,85],"call":[56],"caution,":[57],"as":[58,118],"penalty":[60],"function":[61,88],"added":[62],"to":[63,130],"dual":[65],"objective":[66],"linear":[69],"programming":[70],"(LP)":[71],"formulation":[72],"learning.":[75],"The":[76],"caution":[77],"measures":[78],"distributional":[80],"risk":[81,164],"policy,":[84],"is":[86],"policy's":[91],"long-term":[92],"occupancy":[94],"distribution.":[95],"solve":[97],"problem":[99],"an":[101],"online":[102],"model-free":[103],"manner,":[104],"stochastic":[108],"variant":[109],"primal-dual":[111],"method":[112],"uses":[114],"Kullback-Lieber":[115],"(KL)":[116],"divergence":[117],"its":[119,153],"proximal":[120],"term.":[121],"establish":[123],"number":[126],"iterations/samples":[128],"required":[129],"attain":[131],"approximately":[132],"optimal":[133],"solutions":[134],"scheme":[137],"matches":[138],"tight":[139],"dependencies":[140],"on":[141,155],"cardinality":[143],"spaces,":[149],"but":[150],"differs":[151],"dependence":[154],"infinity":[157],"norm":[158],"gradient":[161],"measure.":[165],"Experiments":[166],"demonstrate":[167],"merits":[169],"approach":[172],"for":[173],"improving":[174],"reliability":[176],"reward":[178],"accumulation":[179],"without":[180],"additional":[181],"burdens.":[183]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}