{"id":"https://openalex.org/W7134281710","doi":"https://doi.org/10.48550/arxiv.2603.06009","title":"Preventing Learning Stagnation in PPO by Scaling to 1 Million Parallel Environments","display_name":"Preventing Learning Stagnation in PPO by Scaling to 1 Million Parallel Environments","publication_year":2026,"publication_date":"2026-03-06","ids":{"openalex":"https://openalex.org/W7134281710","doi":"https://doi.org/10.48550/arxiv.2603.06009"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2603.06009","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091348511","display_name":"Michael Beukman","orcid":"https://orcid.org/0000-0002-5468-284X"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Beukman, Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028021826","display_name":"Khimya Khetarpal","orcid":"https://orcid.org/0000-0001-9975-6438"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Khetarpal, Khimya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128519005","display_name":"Zeyu Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Zeyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037981481","display_name":"Will Dabney","orcid":"https://orcid.org/0000-0003-4600-5520"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dabney, Will","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128562322","display_name":"Jakob Foerster","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Foerster, Jakob","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128588158","display_name":"Michael Dennis","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dennis, Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5089486474","display_name":"Clare Lyle","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lyle, Clare","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5091348511"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.7339000105857849,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.7339000105857849,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.03099999949336052,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.030899999663233757,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hyperparameter","display_name":"Hyperparameter","score":0.7896999716758728},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.6697999835014343},{"id":"https://openalex.org/keywords/regularization","display_name":"Regularization (linguistics)","score":0.4189999997615814},{"id":"https://openalex.org/keywords/recipe","display_name":"Recipe","score":0.3977999985218048},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.3521000146865845},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.3433000147342682},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.3172999918460846}],"concepts":[{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.7896999716758728},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.6697999835014343},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6409000158309937},{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.4189999997615814},{"id":"https://openalex.org/C2778671685","wikidata":"https://www.wikidata.org/wiki/Q219239","display_name":"Recipe","level":2,"score":0.3977999985218048},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.36160001158714294},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3573000133037567},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.3521000146865845},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.3433000147342682},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.31929999589920044},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.3172999918460846},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.3172000050544739},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.30709999799728394},{"id":"https://openalex.org/C187191949","wikidata":"https://www.wikidata.org/wiki/Q1138496","display_name":"Profiling (computer programming)","level":2,"score":0.28600001335144043},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.28529998660087585},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.2759000062942505},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.2687000036239624},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.2605000138282776},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.25760000944137573},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2554999887943268}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2603.06009","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2603.06009","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.06009","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2603.06009","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Plateaus,":[0],"where":[1],"an":[2],"agent's":[3],"performance":[4,147,264,289],"stagnates":[5],"at":[6,150],"a":[7,11,64,151,230,242,273],"suboptimal":[8,152],"level,":[9],"are":[10,176],"common":[12],"problem":[13],"in":[14,30,168,272],"deep":[15],"on-policy":[16],"RL.":[17],"Focusing":[18],"on":[19],"PPO":[20,66,167,279],"due":[21],"to":[22,163,179,235,246,262,280,292],"its":[23],"widespread":[24],"adoption,":[25],"we":[26,81,102,240,267],"show":[27,255],"that":[28,146,174,222,256],"plateaus":[29],"certain":[31],"regimes":[32],"arise":[33],"not":[34],"because":[35,44],"of":[36,47,61,136,184,196,206,226],"known":[37],"exploration,":[38],"capacity,":[39],"or":[40,192],"optimization":[41],"challenges,":[42],"but":[43],"sample-based":[45],"estimates":[46],"the":[48,55,59,77,83,105,122,126,130,134,155,164,189,194,204,215,224,248],"loss":[49],"eventually":[50],"become":[51],"poor":[52],"proxies":[53],"for":[54,244],"true":[56],"objective":[57],"over":[58],"course":[60],"training.":[62],"As":[63],"recap,":[65],"switches":[67],"between":[68,139,199],"sampling":[69],"rollouts":[70],"from":[71],"several":[72],"parallel":[73,227,284],"environments":[74,228],"online":[75],"using":[76],"current":[78],"policy":[79,128,140],"(which":[80],"call":[82],"outer":[84,106,156],"loop)":[85],"and":[86,108,129,209,218,232,254],"performing":[87],"repeated":[88],"minibatch":[89],"SGD":[90],"steps":[91],"against":[92],"this":[93,169,181],"offline":[94],"dataset":[95],"(the":[96],"inner":[97],"loop).":[98],"In":[99],"our":[100,207],"work":[101],"consider":[103],"only":[104],"loop,":[107],"conceptually":[109],"model":[110,144,208],"it":[111,172],"as":[112],"stochastic":[113],"optimization.":[114],"The":[115],"step":[116,157,190,216],"size":[117,158,191,217],"is":[118,159,229],"then":[119],"controlled":[120],"by":[121,133,277],"regularization":[123],"strength":[124],"towards":[125],"previous":[127],"gradient":[131],"noise":[132],"number":[135,195,225],"samples":[137,197],"collected":[138,198],"update":[141,219],"steps.":[142],"This":[143],"predicts":[145],"will":[148],"plateau":[149],"level":[153],"if":[154],"too":[160],"large":[161],"relative":[162],"noise.":[165],"Recasting":[166],"light":[170],"makes":[171],"clear":[173],"there":[175],"two":[177],"ways":[178],"address":[180],"particular":[182],"type":[183],"learning":[185],"stagnation:":[186],"either":[187],"reduce":[188,236],"increase":[193],"updates.":[200],"We":[201],"first":[202],"validate":[203],"predictions":[205],"investigate":[210],"how":[211,245],"hyperparameter":[212],"choices":[213],"influence":[214],"noise,":[220],"concluding":[221],"increasing":[223,252],"simple":[231],"robust":[233],"way":[234],"both":[237],"factors.":[238],"Next,":[239],"propose":[241],"recipe":[243],"co-scale":[247],"other":[249],"hyperparameters":[250],"when":[251],"parallelization,":[253],"incorrectly":[257],"doing":[258],"so":[259],"can":[260],"lead":[261],"severe":[263],"degradation.":[265],"Finally,":[266],"vastly":[268],"outperform":[269],"prior":[270],"baselines":[271],"complex":[274],"open-ended":[275],"domain":[276],"scaling":[278],"more":[281],"than":[282],"1M":[283],"environments,":[285],"thereby":[286],"enabling":[287],"monotonic":[288],"improvement":[290],"up":[291],"one":[293],"trillion":[294],"transitions.":[295]},"counts_by_year":[],"updated_date":"2026-05-05T08:41:31.759640","created_date":"2026-03-10T00:00:00"}