{"id":"https://openalex.org/W3011600109","doi":"https://doi.org/10.1109/cdc40024.2019.9030194","title":"Potential-Based Advice for Stochastic Policy Learning","display_name":"Potential-Based Advice for Stochastic Policy Learning","publication_year":2019,"publication_date":"2019-12-01","ids":{"openalex":"https://openalex.org/W3011600109","doi":"https://doi.org/10.1109/cdc40024.2019.9030194","mag":"3011600109"},"language":"en","primary_location":{"id":"doi:10.1109/cdc40024.2019.9030194","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cdc40024.2019.9030194","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 IEEE 58th Conference on Decision and Control (CDC)","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041353829","display_name":"Baicen Xiao","orcid":null},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Baicen Xiao","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052064870","display_name":"Bhaskar Ramasubramanian","orcid":"https://orcid.org/0000-0002-2166-7838"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bhaskar Ramasubramanian","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004774385","display_name":"Andrew Clark","orcid":"https://orcid.org/0000-0002-5868-6186"},"institutions":[{"id":"https://openalex.org/I107077323","display_name":"Worcester Polytechnic Institute","ror":"https://ror.org/05ejpqr48","country_code":"US","type":"education","lineage":["https://openalex.org/I107077323"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew Clark","raw_affiliation_strings":["Worcester Polytechnic Institute, Worcester, MA, USA"],"affiliations":[{"raw_affiliation_string":"Worcester Polytechnic Institute, Worcester, MA, USA","institution_ids":["https://openalex.org/I107077323"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082305994","display_name":"Hannaneh Hajishirzi","orcid":"https://orcid.org/0000-0002-1055-6657"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hannaneh Hajishirzi","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003489427","display_name":"Linda Bushnell","orcid":"https://orcid.org/0000-0002-8751-2409"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Linda Bushnell","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079723268","display_name":"Radha Poovendran","orcid":"https://orcid.org/0000-0003-0269-8097"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Radha Poovendran","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5041353829"],"corresponding_institution_ids":["https://openalex.org/I201448701"],"apc_list":null,"apc_paid":null,"fwci":0.46085324,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.72914636,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":95},"biblio":{"volume":"17","issue":null,"first_page":"1842","last_page":"1849"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9919999837875366,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.964900016784668,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.8416693806648254},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.7222100496292114},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7189853191375732},{"id":"https://openalex.org/keywords/scheme","display_name":"Scheme (mathematics)","score":0.6025221347808838},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.545541524887085},{"id":"https://openalex.org/keywords/advice","display_name":"Advice (programming)","score":0.5282130837440491},{"id":"https://openalex.org/keywords/grid","display_name":"Grid","score":0.48975062370300293},{"id":"https://openalex.org/keywords/stochastic-control","display_name":"Stochastic control","score":0.47670572996139526},{"id":"https://openalex.org/keywords/stochastic-optimization","display_name":"Stochastic optimization","score":0.4531266689300537},{"id":"https://openalex.org/keywords/jump","display_name":"Jump","score":0.45172399282455444},{"id":"https://openalex.org/keywords/optimal-control","display_name":"Optimal control","score":0.4362286925315857},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.4276338517665863},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.4207664728164673},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.4143497943878174},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3478870987892151},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.16700783371925354},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.13797059655189514},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.07486879825592041}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8416693806648254},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.7222100496292114},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7189853191375732},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.6025221347808838},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.545541524887085},{"id":"https://openalex.org/C2779955035","wikidata":"https://www.wikidata.org/wiki/Q4686785","display_name":"Advice (programming)","level":2,"score":0.5282130837440491},{"id":"https://openalex.org/C187691185","wikidata":"https://www.wikidata.org/wiki/Q2020720","display_name":"Grid","level":2,"score":0.48975062370300293},{"id":"https://openalex.org/C170131372","wikidata":"https://www.wikidata.org/wiki/Q7617811","display_name":"Stochastic control","level":3,"score":0.47670572996139526},{"id":"https://openalex.org/C194387892","wikidata":"https://www.wikidata.org/wiki/Q1747770","display_name":"Stochastic optimization","level":2,"score":0.4531266689300537},{"id":"https://openalex.org/C2780695682","wikidata":"https://www.wikidata.org/wiki/Q4005959","display_name":"Jump","level":2,"score":0.45172399282455444},{"id":"https://openalex.org/C91575142","wikidata":"https://www.wikidata.org/wiki/Q1971426","display_name":"Optimal control","level":2,"score":0.4362286925315857},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.4276338517665863},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.4207664728164673},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.4143497943878174},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3478870987892151},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.16700783371925354},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.13797059655189514},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.07486879825592041},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/cdc40024.2019.9030194","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cdc40024.2019.9030194","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 IEEE 58th Conference on Decision and Control (CDC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":53,"referenced_works":["https://openalex.org/W41554520","https://openalex.org/W121023703","https://openalex.org/W1499408472","https://openalex.org/W1522301498","https://openalex.org/W1777239053","https://openalex.org/W1825869920","https://openalex.org/W1993411524","https://openalex.org/W2061902782","https://openalex.org/W2071983464","https://openalex.org/W2094387729","https://openalex.org/W2116157560","https://openalex.org/W2130750514","https://openalex.org/W2145339207","https://openalex.org/W2151382427","https://openalex.org/W2155007355","https://openalex.org/W2155027007","https://openalex.org/W2164419340","https://openalex.org/W2202549229","https://openalex.org/W2257979135","https://openalex.org/W2620974420","https://openalex.org/W2781726626","https://openalex.org/W2788455270","https://openalex.org/W2808546214","https://openalex.org/W2886474253","https://openalex.org/W2897661175","https://openalex.org/W2912747791","https://openalex.org/W2949475445","https://openalex.org/W2949561945","https://openalex.org/W2963523627","https://openalex.org/W2963864421","https://openalex.org/W2964043796","https://openalex.org/W4298023569","https://openalex.org/W6604963999","https://openalex.org/W6631190155","https://openalex.org/W6638088447","https://openalex.org/W6675999342","https://openalex.org/W6677262939","https://openalex.org/W6682205418","https://openalex.org/W6682849425","https://openalex.org/W6683204974","https://openalex.org/W6684159546","https://openalex.org/W6684921986","https://openalex.org/W6687713970","https://openalex.org/W6692846177","https://openalex.org/W6730641667","https://openalex.org/W6734517396","https://openalex.org/W6738087714","https://openalex.org/W6739055881","https://openalex.org/W6747473740","https://openalex.org/W6747861887","https://openalex.org/W6752543704","https://openalex.org/W6754297394","https://openalex.org/W6780559895"],"related_works":["https://openalex.org/W2943897807","https://openalex.org/W4366198066","https://openalex.org/W3120484221","https://openalex.org/W3047748938","https://openalex.org/W2358522863","https://openalex.org/W4386034604","https://openalex.org/W3099285423","https://openalex.org/W278441094","https://openalex.org/W4381248241","https://openalex.org/W2336173978"],"abstract_inverted_index":{"This":[0],"paper":[1],"augments":[2],"the":[3,18,44,113,131],"reward":[4,30],"received":[5],"by":[6],"a":[7,28,67,105,135,142],"reinforcement":[8],"learning":[9],"agent":[10,19,48,132],"with":[11,87,109],"potential":[12],"functions":[13],"in":[14],"order":[15],"to":[16,35,49,62,69,74,133],"help":[17],"learn":[20,50,134],"(possibly":[21],"stochastic)":[22],"optimal":[23,52,137],"policies.":[24],"We":[25,65],"show":[26],"that":[27,43,80,127],"potential-based":[29,71],"shaping":[31],"scheme":[32,59,89],"is":[33,54,60,90],"able":[34],"preserve":[36],"optimality":[37],"of":[38,46],"stochastic":[39,136],"policies,":[40],"and":[41,92,112,116,140],"demonstrate":[42],"ability":[45],"an":[47,51,82],"policy":[53,75,138],"not":[55],"affected":[56],"when":[57],"this":[58,88],"augmented":[61,86],"soft":[63],"Q-learning.":[64],"propose":[66],"method":[68],"impart":[70],"advice":[72],"schemes":[73,129],"gradient":[76],"algorithms.":[77],"An":[78],"algorithm":[79],"considers":[81],"advantage":[83],"actor-critic":[84],"architecture":[85],"proposed,":[91],"we":[93,100],"give":[94],"guarantees":[95],"on":[96,104],"its":[97],"convergence.":[98],"Finally,":[99],"evaluate":[101],"our":[102],"approach":[103],"puddle-jump":[106],"grid":[107],"world":[108],"indistinguishable":[110],"states,":[111],"continuous":[114],"state":[115],"action":[117],"mountain":[118],"car":[119],"environment":[120],"from":[121],"classical":[122],"control.":[123],"Our":[124],"results":[125],"indicate":[126],"these":[128],"allow":[130],"faster":[139],"obtain":[141],"higher":[143],"average":[144],"reward.":[145]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}