{"id":"https://openalex.org/W7127622930","doi":"https://doi.org/10.48550/arxiv.2602.03386","title":"An Approximate Ascent Approach To Prove Convergence of PPO","display_name":"An Approximate Ascent Approach To Prove Convergence of PPO","publication_year":2026,"publication_date":"2026-02-03","ids":{"openalex":"https://openalex.org/W7127622930","doi":"https://doi.org/10.48550/arxiv.2602.03386"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2602.03386","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2602.03386","pdf_url":"https://arxiv.org/pdf/2602.03386","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2602.03386","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048798373","display_name":"Leif D\u00f6ring","orcid":"https://orcid.org/0000-0002-4569-5083","observed_orcids":["https://orcid.org/0000-0002-4569-5083"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Doering, Leif","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125008298","display_name":"Daniel Schmidt","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I112756935","display_name":"Technische Hochschule Mannheim","ror":"https://ror.org/04p61dj41","country_code":"DE","type":"education","lineage":["https://openalex.org/I112756935"]},{"id":"https://openalex.org/I177802217","display_name":"University of Mannheim","ror":"https://ror.org/031bsb921","country_code":"DE","type":"education","lineage":["https://openalex.org/I177802217"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Schmidt, Daniel","raw_affiliation_strings":["Institute of Mathematics , University of Mannheim , 68138 Mannheim , Germany,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Mathematics , University of Mannheim , 68138 Mannheim , Germany,","institution_ids":["https://openalex.org/I112756935","https://openalex.org/I177802217"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119845504","display_name":"Moritz Melcher","orcid":"https://orcid.org/0009-0005-7182-7173","observed_orcids":["https://orcid.org/0009-0005-7182-7173"]},"institutions":[{"id":"https://openalex.org/I112756935","display_name":"Technische Hochschule Mannheim","ror":"https://ror.org/04p61dj41","country_code":"DE","type":"education","lineage":["https://openalex.org/I112756935"]},{"id":"https://openalex.org/I177802217","display_name":"University of Mannheim","ror":"https://ror.org/031bsb921","country_code":"DE","type":"education","lineage":["https://openalex.org/I177802217"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Melcher, Moritz","raw_affiliation_strings":["Institute of Mathematics , University of Mannheim , 68138 Mannheim , Germany,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Mathematics , University of Mannheim , 68138 Mannheim , Germany,","institution_ids":["https://openalex.org/I112756935","https://openalex.org/I177802217"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045569270","display_name":"Sebastian Kassing","orcid":"https://orcid.org/0000-0002-2016-899X","observed_orcids":["https://orcid.org/0000-0002-2016-899X"]},"institutions":[{"id":"https://openalex.org/I167360494","display_name":"University of Wuppertal","ror":"https://ror.org/00613ak93","country_code":"DE","type":"education","lineage":["https://openalex.org/I167360494"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Kassing, Sebastian","raw_affiliation_strings":["Department of Mathematics & Informatics , University of Wuppertal , 42119 Wuppertal , Germany,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Mathematics & Informatics , University of Wuppertal , 42119 Wuppertal , Germany,","institution_ids":["https://openalex.org/I167360494"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125066004","display_name":"Benedikt Wille","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I112756935","display_name":"Technische Hochschule Mannheim","ror":"https://ror.org/04p61dj41","country_code":"DE","type":"education","lineage":["https://openalex.org/I112756935"]},{"id":"https://openalex.org/I177802217","display_name":"University of Mannheim","ror":"https://ror.org/031bsb921","country_code":"DE","type":"education","lineage":["https://openalex.org/I177802217"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Wille, Benedikt","raw_affiliation_strings":["Institute of Mathematics , University of Mannheim , 68138 Mannheim , Germany,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Mathematics , University of Mannheim , 68138 Mannheim , Germany,","institution_ids":["https://openalex.org/I112756935","https://openalex.org/I177802217"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125018740","display_name":"Tilman Aach","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I112756935","display_name":"Technische Hochschule Mannheim","ror":"https://ror.org/04p61dj41","country_code":"DE","type":"education","lineage":["https://openalex.org/I112756935"]},{"id":"https://openalex.org/I177802217","display_name":"University of Mannheim","ror":"https://ror.org/031bsb921","country_code":"DE","type":"education","lineage":["https://openalex.org/I177802217"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Aach, Tilman","raw_affiliation_strings":["Institute of Mathematics , University of Mannheim , 68138 Mannheim , Germany,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Mathematics , University of Mannheim , 68138 Mannheim , Germany,","institution_ids":["https://openalex.org/I112756935","https://openalex.org/I177802217"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074701373","display_name":"Simon Wei\u00dfmann","orcid":"https://orcid.org/0000-0002-5111-6658","observed_orcids":["https://orcid.org/0000-0002-5111-6658"]},"institutions":[{"id":"https://openalex.org/I112756935","display_name":"Technische Hochschule Mannheim","ror":"https://ror.org/04p61dj41","country_code":"DE","type":"education","lineage":["https://openalex.org/I112756935"]},{"id":"https://openalex.org/I177802217","display_name":"University of Mannheim","ror":"https://ror.org/031bsb921","country_code":"DE","type":"education","lineage":["https://openalex.org/I177802217"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Weissmann, Simon","raw_affiliation_strings":["Institute of Mathematics , University of Mannheim , 68138 Mannheim , Germany,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Mathematics , University of Mannheim , 68138 Mannheim , Germany,","institution_ids":["https://openalex.org/I112756935","https://openalex.org/I177802217"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11195","display_name":"Simulation Techniques and Applications","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/proximal-policy-optimization","display_name":"proximal policy optimization","score":0.8989999890327454},{"id":"https://openalex.org/keywords/convergence-analysis","display_name":"convergence analysis","score":0.8889999985694885},{"id":"https://openalex.org/keywords/random-reshuffling","display_name":"random reshuffling","score":0.9290000200271606},{"id":"https://openalex.org/keywords/generalized-advantage-estimation","display_name":"generalized advantage estimation","score":0.7440000176429749},{"id":"https://openalex.org/keywords/deep-reinforcement-learning","display_name":"deep reinforcement learning","score":0.7509999871253967}],"concepts":[{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.6847000122070312},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.6582000255584717},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.6230999827384949},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5378999710083008},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.5163000226020813},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.5135999917984009},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.4779999852180481},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.4383000135421753},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.42489999532699585},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.4237000048160553},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.4140999913215637},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.39160001277923584},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.3465999960899353},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.3361000120639801},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.30059999227523804},{"id":"https://openalex.org/C148043351","wikidata":"https://www.wikidata.org/wiki/Q4456944","display_name":"Current (fluid)","level":2,"score":0.29670000076293945},{"id":"https://openalex.org/C134121241","wikidata":"https://www.wikidata.org/wiki/Q899301","display_name":"Yield (engineering)","level":2,"score":0.2777000069618225},{"id":"https://openalex.org/C70136482","wikidata":"https://www.wikidata.org/wiki/Q13583781","display_name":"A-weighting","level":3,"score":0.2678000032901764},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.266400009393692},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.2653999924659729},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.26249998807907104},{"id":"https://openalex.org/C95763700","wikidata":"https://www.wikidata.org/wiki/Q578985","display_name":"Convergence of random variables","level":3,"score":0.25940001010894775},{"id":"https://openalex.org/C2985420695","wikidata":"https://www.wikidata.org/wiki/Q175199","display_name":"Population variance","level":3,"score":0.25189998745918274}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2602.03386","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2602.03386","pdf_url":"https://arxiv.org/pdf/2602.03386","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2602.03386","doi":"https://doi.org/10.48550/arxiv.2602.03386","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.03386","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2602.03386","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2602.03386","pdf_url":"https://arxiv.org/pdf/2602.03386","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7127622930.pdf","grobid_xml":"https://content.openalex.org/works/W7127622930.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3107926684","https://openalex.org/W2965438240","https://openalex.org/W4297900582","https://openalex.org/W3210239845","https://openalex.org/W7095008446","https://openalex.org/W3094535073","https://openalex.org/W6966449329","https://openalex.org/W2970851609","https://openalex.org/W4321500856","https://openalex.org/W7108319380"],"abstract_inverted_index":{"Proximal":[0],"Policy":[1],"Optimization":[2],"(PPO)":[3],"is":[4],"among":[5],"the":[6,69,73,120],"most":[7],"widely":[8,30],"used":[9,108],"deep":[10],"reinforcement":[11],"learning":[12],"algorithms,":[13],"yet":[14],"its":[15],"theoretical":[16],"foundations":[17],"remain":[18,29],"incomplete.":[19],"Most":[20],"importantly,":[21],"convergence":[22,85],"and":[23,76],"understanding":[24],"of":[25,46],"fundamental":[26],"PPO":[27,88],"advantages":[28],"open.":[31],"Under":[32],"standard":[33],"theory":[34],"assumptions":[35],"we":[36,96],"show":[37,65,130],"how":[38,66],"PPO's":[39,93],"policy":[40,61],"update":[41],"scheme":[42,114],"(performing":[43],"multiple":[44],"epochs":[45],"minibatch":[47],"updates":[48],"on":[49,92],"multi-use":[50],"rollouts":[51],"with":[52,142],"a":[53,84,98,132],"surrogate":[54,74],"gradient)":[55],"can":[56,136],"be":[57],"interpreted":[58],"as":[59,147],"approximated":[60],"gradient":[62],"ascent.":[63],"We":[64],"to":[67,82],"control":[68],"bias":[70],"accumulated":[71],"by":[72],"gradients":[75],"use":[77],"techniques":[78],"from":[79],"random":[80],"reshuffling":[81],"prove":[83],"theorem":[86],"for":[87],"that":[89,131],"sheds":[90],"light":[91],"success.":[94],"Additionally,":[95],"identify":[97],"previously":[99],"overlooked":[100],"issue":[101],"in":[102,109,140],"truncated":[103],"Generalized":[104],"Advantage":[105],"Estimation":[106],"commonly":[107],"PPO.":[110],"The":[111],"geometric":[112],"weighting":[113],"induces":[115],"infinite":[116],"mass":[117],"collapse":[118],"onto":[119],"longest":[121],"$k$-step":[122],"advantage":[123],"estimator":[124],"at":[125],"episode":[126],"boundaries.":[127],"Empirical":[128],"evaluations":[129],"simple":[133],"weight":[134],"correction":[135],"yield":[137],"substantial":[138],"improvements":[139],"environments":[141],"strong":[143],"terminal":[144],"signal,":[145],"such":[146],"Lunar":[148],"Lander.":[149]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-02-06T00:00:00"}