{"id":"https://openalex.org/W7140172760","doi":"https://doi.org/10.48550/arxiv.2603.20521","title":"Delightful Distributed Policy Gradient","display_name":"Delightful Distributed Policy Gradient","publication_year":2026,"publication_date":"2026-03-20","ids":{"openalex":"https://openalex.org/W7140172760","doi":"https://doi.org/10.48550/arxiv.2603.20521"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.20521","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.20521","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.20521","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Osband, Ian","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Osband, Ian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.7059000134468079,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.7059000134468079,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.06800000369548798,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.065700002014637,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7001000046730042},{"id":"https://openalex.org/keywords/mnist-database","display_name":"MNIST database","score":0.6176000237464905},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5697000026702881},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.4871000051498413},{"id":"https://openalex.org/keywords/core","display_name":"Core (optical fiber)","score":0.43939998745918274},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.43470001220703125}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7001000046730042},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6205000281333923},{"id":"https://openalex.org/C190502265","wikidata":"https://www.wikidata.org/wiki/Q17069496","display_name":"MNIST database","level":3,"score":0.6176000237464905},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5697000026702881},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4871000051498413},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47290000319480896},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.43939998745918274},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.43470001220703125},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.3962000012397766},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3603000044822693},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.3544999957084656},{"id":"https://openalex.org/C2779436431","wikidata":"https://www.wikidata.org/wiki/Q30672407","display_name":"Policy learning","level":2,"score":0.3188000023365021},{"id":"https://openalex.org/C190839683","wikidata":"https://www.wikidata.org/wiki/Q2448197","display_name":"Train","level":2,"score":0.27950000762939453},{"id":"https://openalex.org/C194544171","wikidata":"https://www.wikidata.org/wiki/Q21105679","display_name":"Gating","level":2,"score":0.2653999924659729},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.26350000500679016},{"id":"https://openalex.org/C2779582901","wikidata":"https://www.wikidata.org/wiki/Q21013010","display_name":"Distributed learning","level":2,"score":0.25519999861717224},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.25040000677108765}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.20521","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.20521","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.20521","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.20521","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.8072046637535095,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Distributed":[0],"reinforcement":[1],"learning":[2,34],"trains":[3],"on":[4],"data":[5,29],"from":[6,35],"stale,":[7],"buggy,":[8],"or":[9],"mismatched":[10],"actors,":[11],"producing":[12],"actions":[13],"with":[14,71,133,143,152,181],"high":[15],"surprisal":[16],"(negative":[17],"log-probability)":[18],"under":[19],"the":[20,53,73,95,109],"learner's":[21],"policy.":[22],"The":[23,59,112],"core":[24],"difficulty":[25],"is":[26,115,177],"not":[27],"surprising":[28,36],"per":[30],"se,":[31],"but":[32],"\\emph{negative":[33],"data}.":[37],"High-surprisal":[38],"failures":[39,81,101,126],"can":[40],"dominate":[41],"finite-batch":[42],"updates":[43],"through":[44],"large":[45],"perpendicular":[46,96],"components,":[47],"while":[48],"high-surprisal":[49,100],"successes":[50,85],"reveal":[51],"opportunities":[52],"current":[54],"policy":[55],"would":[56],"otherwise":[57],"miss.":[58],"\\textit{Delightful":[60],"Policy":[61],"Gradient}":[62],"(DG)":[63],"separates":[64],"these":[65],"cases":[66],"by":[67,102],"gating":[68],"each":[69],"update":[70],"delight,":[72],"product":[74],"of":[75,99],"advantage":[76,113,176],"and":[77,82,158,179],"surprisal,":[78],"suppressing":[79],"rare":[80,84,125,129,159],"preserving":[83],"without":[86,137],"behavior":[87,145],"probabilities.":[88,146],"In":[89],"a":[90,103,148],"tabular":[91],"analysis,":[92],"DG":[93,136,161],"suppresses":[94,124,128],"second":[97],"moment":[98],"policy-overlap":[104],"factor":[105],"that":[106,123],"vanishes":[107],"as":[108],"learner":[110],"improves.":[111],"sign":[114],"essential":[116],"for":[117],"surprisal-based":[118],"filtering:":[119],"any":[120],"learner-probability-only":[121],"gate":[122],"also":[127],"successes.":[130],"On":[131,147],"MNIST":[132],"simulated":[134],"staleness,":[135,153],"off-policy":[138],"correction":[139],"outperforms":[140],"importance-weighted":[141],"PG":[142],"exact":[144],"transformer":[149],"sequence":[150],"task":[151,182],"actor":[154],"bugs,":[155],"reward":[156],"corruption,":[157],"discovery,":[160],"often":[162],"achieves":[163],"nearly":[164],"order-of-magnitude":[165,178],"lower":[166],"error.":[167],"When":[168],"all":[169],"four":[170],"frictions":[171],"act":[172],"simultaneously,":[173],"its":[174],"sample-efficiency":[175],"grows":[180],"complexity.":[183]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-03-25T00:00:00"}