{"id":"https://openalex.org/W7127622930","doi":"https://doi.org/10.48550/arxiv.2602.03386","title":"An Approximate Ascent Approach To Prove Convergence of PPO","display_name":"An Approximate Ascent Approach To Prove Convergence of PPO","publication_year":2026,"publication_date":"2026-02-03","ids":{"openalex":"https://openalex.org/W7127622930","doi":"https://doi.org/10.48550/arxiv.2602.03386"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.03386","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5110618161","display_name":"Leif Doering","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Doering, Leif","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125008298","display_name":"Daniel Schmidt","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Schmidt, Daniel","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119845504","display_name":"Moritz Melcher","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Melcher, Moritz","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045569270","display_name":"Sebastian Kassing","orcid":"https://orcid.org/0000-0002-2016-899X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kassing, Sebastian","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125066004","display_name":"Benedikt Wille","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wille, Benedikt","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125018740","display_name":"Tilman Aach","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aach, Tilman","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5074701373","display_name":"Simon Wei\u00dfmann","orcid":"https://orcid.org/0000-0002-5111-6658"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Weissmann, Simon","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5110618161"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.564300000667572,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.564300000667572,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.1185000017285347,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.04039999842643738,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.6847000122070312},{"id":"https://openalex.org/keywords/weighting","display_name":"Weighting","score":0.6582000255584717},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.6230999827384949},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.4779999852180481},{"id":"https://openalex.org/keywords/scheme","display_name":"Scheme (mathematics)","score":0.4383000135421753},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.42489999532699585},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.4237000048160553},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.4140999913215637}],"concepts":[{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.6847000122070312},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.6582000255584717},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.6230999827384949},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5378999710083008},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.5163000226020813},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.5135999917984009},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.4779999852180481},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.4383000135421753},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.42489999532699585},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.4237000048160553},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.4140999913215637},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.39160001277923584},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.3465999960899353},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.3361000120639801},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.30059999227523804},{"id":"https://openalex.org/C148043351","wikidata":"https://www.wikidata.org/wiki/Q4456944","display_name":"Current (fluid)","level":2,"score":0.29670000076293945},{"id":"https://openalex.org/C134121241","wikidata":"https://www.wikidata.org/wiki/Q899301","display_name":"Yield (engineering)","level":2,"score":0.2777000069618225},{"id":"https://openalex.org/C70136482","wikidata":"https://www.wikidata.org/wiki/Q13583781","display_name":"A-weighting","level":3,"score":0.2678000032901764},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.266400009393692},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.2653999924659729},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.26249998807907104},{"id":"https://openalex.org/C95763700","wikidata":"https://www.wikidata.org/wiki/Q578985","display_name":"Convergence of random variables","level":3,"score":0.25940001010894775},{"id":"https://openalex.org/C2985420695","wikidata":"https://www.wikidata.org/wiki/Q175199","display_name":"Population variance","level":3,"score":0.25189998745918274}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.03386","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.03386","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.03386","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.03386","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.4308375120162964}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Proximal":[0],"Policy":[1],"Optimization":[2],"(PPO)":[3],"is":[4],"among":[5],"the":[6,69,73,120],"most":[7],"widely":[8,30],"used":[9,108],"deep":[10],"reinforcement":[11],"learning":[12],"algorithms,":[13],"yet":[14],"its":[15],"theoretical":[16],"foundations":[17],"remain":[18,29],"incomplete.":[19],"Most":[20],"importantly,":[21],"convergence":[22,85],"and":[23,76],"understanding":[24],"of":[25,46],"fundamental":[26],"PPO":[27,88],"advantages":[28],"open.":[31],"Under":[32],"standard":[33],"theory":[34],"assumptions":[35],"we":[36,96],"show":[37,65,130],"how":[38,66],"PPO's":[39,93],"policy":[40,61],"update":[41],"scheme":[42,114],"(performing":[43],"multiple":[44],"epochs":[45],"minibatch":[47],"updates":[48],"on":[49,92],"multi-use":[50],"rollouts":[51],"with":[52,142],"a":[53,84,98,132],"surrogate":[54,74],"gradient)":[55],"can":[56,136],"be":[57],"interpreted":[58],"as":[59,147],"approximated":[60],"gradient":[62],"ascent.":[63],"We":[64],"to":[67,82],"control":[68],"bias":[70],"accumulated":[71],"by":[72],"gradients":[75],"use":[77],"techniques":[78],"from":[79],"random":[80],"reshuffling":[81],"prove":[83],"theorem":[86],"for":[87],"that":[89,131],"sheds":[90],"light":[91],"success.":[94],"Additionally,":[95],"identify":[97],"previously":[99],"overlooked":[100],"issue":[101],"in":[102,109,140],"truncated":[103],"Generalized":[104],"Advantage":[105],"Estimation":[106],"commonly":[107],"PPO.":[110],"The":[111],"geometric":[112],"weighting":[113],"induces":[115],"infinite":[116],"mass":[117],"collapse":[118],"onto":[119],"longest":[121],"$k$-step":[122],"advantage":[123],"estimator":[124],"at":[125],"episode":[126],"boundaries.":[127],"Empirical":[128],"evaluations":[129],"simple":[133],"weight":[134],"correction":[135],"yield":[137],"substantial":[138],"improvements":[139],"environments":[141],"strong":[143],"terminal":[144],"signal,":[145],"such":[146],"Lunar":[148],"Lander.":[149]},"counts_by_year":[],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2026-02-06T00:00:00"}