{"id":"https://openalex.org/W6990719774","doi":"https://doi.org/10.48550/arxiv.1004.2027","title":"Dynamic Policy Programming","display_name":"Dynamic Policy Programming","publication_year":2010,"publication_date":"2010-04-12","ids":{"openalex":"https://openalex.org/W6990719774","doi":"https://doi.org/10.48550/arxiv.1004.2027","mag":"2962901215"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:1004.2027","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/1004.2027","pdf_url":"https://arxiv.org/pdf/1004.2027","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1004.2027","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5142129447","display_name":"M. Gheshlaghi Azar","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I145872427","display_name":"Radboud University Nijmegen","ror":"https://ror.org/016xsfp80","country_code":"NL","type":"education","lineage":["https://openalex.org/I145872427"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Azar, Mohammad Gheshlaghi","raw_affiliation_strings":["Department of Biophysics Radboud University Nijmegen 6525 EZ Nijmegen , The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Biophysics Radboud University Nijmegen 6525 EZ Nijmegen , The Netherlands","institution_ids":["https://openalex.org/I145872427"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5151315056","display_name":"V. Gomez","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I145872427","display_name":"Radboud University Nijmegen","ror":"https://ror.org/016xsfp80","country_code":"NL","type":"education","lineage":["https://openalex.org/I145872427"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Gomez, Vicenc","raw_affiliation_strings":["Department of Biophysics Radboud University Nijmegen 6525 EZ Nijmegen , The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Biophysics Radboud University Nijmegen 6525 EZ Nijmegen , The Netherlands","institution_ids":["https://openalex.org/I145872427"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5050819882","display_name":"Hilbert J. Kappen","orcid":"https://orcid.org/0000-0002-5728-3676","observed_orcids":["https://orcid.org/0000-0002-5728-3676"]},"institutions":[{"id":"https://openalex.org/I145872427","display_name":"Radboud University Nijmegen","ror":"https://ror.org/016xsfp80","country_code":"NL","type":"education","lineage":["https://openalex.org/I145872427"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Kappen, Hilbert J.","raw_affiliation_strings":["Department of Biophysics Radboud University Nijmegen 6525 EZ Nijmegen , The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Biophysics Radboud University Nijmegen 6525 EZ Nijmegen , The Netherlands","institution_ids":["https://openalex.org/I145872427"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I145872427"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":94,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"13","issue":"1","first_page":"3207","last_page":"3245"},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.010200000368058681,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11195","display_name":"Simulation Techniques and Applications","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/policy-iteration","display_name":"policy iteration","score":0.875},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.8870000243186951},{"id":"https://openalex.org/keywords/performance-bounds","display_name":"performance bounds","score":0.5149999856948853},{"id":"https://openalex.org/keywords/approximation-error","display_name":"approximation error","score":0.7739999890327454},{"id":"https://openalex.org/keywords/approximate-value-iteration","display_name":"approximate value iteration","score":0.8019999861717224},{"id":"https://openalex.org/keywords/approximate-policy-iteration","display_name":"approximate policy iteration","score":0.9990000128746033},{"id":"https://openalex.org/keywords/reinforcement-learning-algorithms","display_name":"reinforcement learning algorithms","score":0.6629999876022339}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6147000193595886},{"id":"https://openalex.org/C43169469","wikidata":"https://www.wikidata.org/wiki/Q580922","display_name":"Preprint","level":2,"score":0.47189998626708984},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.32409998774528503},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.30649998784065247},{"id":"https://openalex.org/C37404715","wikidata":"https://www.wikidata.org/wiki/Q380679","display_name":"Dynamic programming","level":2,"score":0.2973000109195709},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.290800005197525},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.288100004196167},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.28139999508857727},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.27230000495910645},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.2630999982357025}],"mesh":[],"locations_count":9,"locations":[{"id":"pmh:oai:arXiv.org:1004.2027","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/1004.2027","pdf_url":"https://arxiv.org/pdf/1004.2027","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:repository.ubn.ru.nl:2066/110989","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/2066/110989","pdf_url":"http://hdl.handle.net/2066/110989","source":{"id":"https://openalex.org/S4306401067","display_name":"Radboud Repository (Radboud University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I145872427","host_organization_name":"Radboud University Nijmegen","host_organization_lineage":["https://openalex.org/I145872427"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://jmlr.csail.mit.edu/proceedings/papers/v15/azar11a.html","raw_type":"Article / Letter to editor"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.1017.4207","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.1017.4207","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://repository.ubn.ru.nl/bitstream/handle/2066/110989/110989.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.297.3915","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.297.3915","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.mbfys.ru.nl/staff/v.gomez/azar12a.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.762.4194","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.762.4194","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://arxiv.org/pdf/1004.2027.pdf","raw_type":"text"},{"id":"pmh:ru:oai:repository.ubn.ru.nl:2066/110989","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/2066/110989","pdf_url":null,"source":{"id":"https://openalex.org/S4306401067","display_name":"Radboud Repository (Radboud University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I145872427","host_organization_name":"Radboud University Nijmegen","host_organization_lineage":["https://openalex.org/I145872427"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Journal of Machine Learning Research, 3207 - 3245","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:ru:oai:repository.ubn.ru.nl:2066/88822","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/2066/88822","pdf_url":null,"source":{"id":"https://openalex.org/S4306401067","display_name":"Radboud Repository (Radboud University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I145872427","host_organization_name":"Radboud University Nijmegen","host_organization_lineage":["https://openalex.org/I145872427"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Journal of Machine Learning Research, arXiv, 1 - 26","raw_type":"info:eu-repo/semantics/article"},{"id":"doi:10.48550/arxiv.1004.2027","doi":"https://doi.org/10.48550/arxiv.1004.2027","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1004.2027","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"mag:2962901215","doi":null,"is_oa":false,"landing_page_url":"https://jmlr.csail.mit.edu/papers/volume13/azar12a/azar12a.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S118988714","display_name":"Journal of Machine Learning Research","issn_l":"1532-4435","issn":["1532-4435","1533-7928"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","ccf-a","cwts-core","dongbi-a","jufo-3","ki-jl-2","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315718","host_organization_name":"The MIT Press","host_organization_lineage":["https://openalex.org/P4310315718","https://openalex.org/P4310316440"],"host_organization_lineage_names":["The MIT Press","Massachusetts Institute of Technology"],"type":"journal"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"Journal of Machine Learning Research","raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1004.2027","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/1004.2027","pdf_url":"https://arxiv.org/pdf/1004.2027","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W6990719774.pdf","grobid_xml":"https://content.openalex.org/works/W6990719774.grobid-xml"},"referenced_works_count":56,"referenced_works":["https://openalex.org/W13294968","https://openalex.org/W114364418","https://openalex.org/W1499669280","https://openalex.org/W1515851193","https://openalex.org/W1526654727","https://openalex.org/W1541730457","https://openalex.org/W1570963478","https://openalex.org/W1576452626","https://openalex.org/W1583072328","https://openalex.org/W1583833196","https://openalex.org/W1662803991","https://openalex.org/W1850488217","https://openalex.org/W2009303086","https://openalex.org/W2013164703","https://openalex.org/W2013614847","https://openalex.org/W2020920737","https://openalex.org/W2026659355","https://openalex.org/W2071983464","https://openalex.org/W2073384958","https://openalex.org/W2086602917","https://openalex.org/W2094387729","https://openalex.org/W2096351669","https://openalex.org/W2098432798","https://openalex.org/W2105336508","https://openalex.org/W2107662876","https://openalex.org/W2107977784","https://openalex.org/W2111967991","https://openalex.org/W2117355432","https://openalex.org/W2118686230","https://openalex.org/W2120346334","https://openalex.org/W2121863487","https://openalex.org/W2122701159","https://openalex.org/W2123447947","https://openalex.org/W2124175081","https://openalex.org/W2128812357","https://openalex.org/W2130005627","https://openalex.org/W2130801532","https://openalex.org/W2144446635","https://openalex.org/W2145060720","https://openalex.org/W2149166950","https://openalex.org/W2149418961","https://openalex.org/W2150147323","https://openalex.org/W2150339816","https://openalex.org/W2151416233","https://openalex.org/W2151661095","https://openalex.org/W2155027007","https://openalex.org/W2158191646","https://openalex.org/W2158738729","https://openalex.org/W2165131254","https://openalex.org/W2167117957","https://openalex.org/W2167999092","https://openalex.org/W2172425052","https://openalex.org/W2172968643","https://openalex.org/W2305373430","https://openalex.org/W3140968660","https://openalex.org/W6884728263"],"related_works":["https://openalex.org/W7094781866","https://openalex.org/W7009326675","https://openalex.org/W3176888410","https://openalex.org/W4210255130","https://openalex.org/W2010718801","https://openalex.org/W1974989180","https://openalex.org/W6903847790","https://openalex.org/W3152730573","https://openalex.org/W2471352553","https://openalex.org/W388384896"],"abstract_inverted_index":{"In":[0],"this":[1,111],"paper,":[2],"we":[3],"propose":[4],"a":[5,87,150],"novel":[6],"policy":[7,12,19,78],"iteration":[8,73,79],"method,":[9],"called":[10],"dynamic":[11],"programming":[13],"(DPP),":[14],"to":[15,59],"estimate":[16],"the":[17,21,28,38,50,53,60,63,66,69,76,98,106,118,121],"optimal":[18],"in":[20,37,47,65,140],"infinite-horizon":[22],"Markov":[23],"decision":[24],"processes.":[25],"We":[26,109],"prove":[27],"finite-iteration":[29],"and":[30,75,92],"asymptotic":[31],"l\\infty-norm":[32,51,61],"performance-loss":[33],"bounds":[34,44],"for":[35],"DPP":[36,84,125],"presence":[39],"of":[40,49,52,62,68,120,124],"approximation/estimation":[41],"error.":[42],"The":[43],"are":[45],"expressed":[46],"terms":[48],"average":[54],"accumulated":[55],"error":[56,64],"as":[57],"opposed":[58],"case":[67],"standard":[70],"approximate":[71,77,122],"value":[72],"(AVI)":[74],"(API).":[80],"This":[81],"suggests":[82],"that":[83],"can":[85],"achieve":[86],"better":[88],"performance":[89,119],"than":[90],"AVI":[91],"API":[93],"since":[94],"it":[95],"averages":[96],"out":[97],"simulation":[99],"noise":[100],"caused":[101],"by":[102,115,149],"Monte-Carlo":[103],"sampling":[104],"throughout":[105],"learning":[107,129],"process.":[108],"examine":[110],"theoretical":[112],"results":[113,137],"numerically":[114],"com-":[116],"paring":[117],"variants":[123],"with":[126],"existing":[127],"reinforcement":[128],"(RL)":[130],"methods":[131,148],"on":[132],"different":[133],"problem":[134],"domains.":[135],"Our":[136],"show":[138],"that,":[139],"all":[141],"cases,":[142],"DPP-based":[143],"algorithms":[144],"outperform":[145],"other":[146],"RL":[147],"wide":[151],"margin.":[152]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":14},{"year":2020,"cited_by_count":16},{"year":2019,"cited_by_count":11},{"year":2018,"cited_by_count":6},{"year":2017,"cited_by_count":8},{"year":2016,"cited_by_count":5},{"year":2015,"cited_by_count":5},{"year":2014,"cited_by_count":6},{"year":2013,"cited_by_count":1},{"year":2012,"cited_by_count":2}],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2025-10-10T00:00:00"}