{"id":"https://openalex.org/W7165773020","doi":"https://doi.org/10.48550/arxiv.2606.23932","title":"KLip-PPO: A per-sample KL perspective on PPO-Clip","display_name":"KLip-PPO: A per-sample KL perspective on PPO-Clip","publication_year":2026,"publication_date":"2026-06-22","ids":{"openalex":"https://openalex.org/W7165773020","doi":"https://doi.org/10.48550/arxiv.2606.23932"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.23932","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.23932","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.23932","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139257454","display_name":"Riccardo Colletti","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Colletti, Riccardo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139239428","display_name":"Robin Holzinger","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Holzinger, Robin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8996999859809875,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8996999859809875,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.019500000402331352,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.01209999993443489,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.6507999897003174},{"id":"https://openalex.org/keywords/notation","display_name":"Notation","score":0.6218000054359436},{"id":"https://openalex.org/keywords/boundary","display_name":"Boundary (topology)","score":0.5953999757766724},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5386999845504761},{"id":"https://openalex.org/keywords/sketch","display_name":"Sketch","score":0.5376999974250793},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.4918999969959259},{"id":"https://openalex.org/keywords/identity","display_name":"Identity (music)","score":0.43689998984336853},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.39899998903274536}],"concepts":[{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.6507999897003174},{"id":"https://openalex.org/C45357846","wikidata":"https://www.wikidata.org/wiki/Q2001982","display_name":"Notation","level":2,"score":0.6218000054359436},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.6187000274658203},{"id":"https://openalex.org/C62354387","wikidata":"https://www.wikidata.org/wiki/Q875399","display_name":"Boundary (topology)","level":2,"score":0.5953999757766724},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5386999845504761},{"id":"https://openalex.org/C2779231336","wikidata":"https://www.wikidata.org/wiki/Q7534724","display_name":"Sketch","level":2,"score":0.5376999974250793},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4918999969959259},{"id":"https://openalex.org/C2778355321","wikidata":"https://www.wikidata.org/wiki/Q17079427","display_name":"Identity (music)","level":2,"score":0.43689998984336853},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.40639999508857727},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.39899998903274536},{"id":"https://openalex.org/C6180225","wikidata":"https://www.wikidata.org/wiki/Q3411771","display_name":"Penalty method","level":2,"score":0.3968999981880188},{"id":"https://openalex.org/C144237770","wikidata":"https://www.wikidata.org/wiki/Q747534","display_name":"Mathematical economics","level":1,"score":0.3831000030040741},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.3709999918937683},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.3693000078201294},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3325999975204468},{"id":"https://openalex.org/C89109886","wikidata":"https://www.wikidata.org/wiki/Q1535924","display_name":"Trust region","level":3,"score":0.3230000138282776},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.31220000982284546},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.3107999861240387},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30390000343322754},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.2921999990940094},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.29100000858306885},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2782999873161316},{"id":"https://openalex.org/C162307627","wikidata":"https://www.wikidata.org/wiki/Q204833","display_name":"Enhanced Data Rates for GSM Evolution","level":2,"score":0.2671999931335449},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.2574000060558319},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.25679999589920044},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.25619998574256897}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.23932","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.23932","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.23932","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.23932","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.74784255027771,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Proximal":[0],"Policy":[1],"Optimization":[2],"(PPO)":[3],"is":[4,74,141,158],"the":[5,25,68,71,90,94,105,115,129,133,146,149,153,159,165,169,174],"standard":[6],"policy-gradient":[7],"algorithm":[8],"for":[9,163],"on-policy":[10],"reinforcement":[11],"learning.":[12],"The":[13,96,122],"literature":[14],"presents":[15],"it":[16],"in":[17,173],"two":[18,116],"forms,":[19],"a":[20,32,57,78,125,142],"clipped":[21,72,130],"surrogate":[22,73,80,131],"that":[23,67,132],"bounds":[24],"importance":[26,91],"ratio":[27,92],"between":[28,35],"successive":[29],"policies":[30],"and":[31,51,56,93,103,109,152],"Kullback-Leibler":[33,79],"penalty":[34,140],"them.":[36,64],"These":[37],"forms":[38],"are":[39],"treated":[40],"as":[41],"separate":[42],"algorithms":[43],"with":[44,86],"their":[45,48,52],"own":[46,49,53],"gradients,":[47],"hyperparameters,":[50],"reference":[54],"implementations,":[55],"sizeable":[58],"body":[59],"of":[60,70,128,148,155],"empirical":[61],"work":[62],"compares":[63],"We":[65,167],"show":[66],"gradient":[69],"reproduced":[75],"exactly":[76],"by":[77],"whose":[81],"coefficient":[82,157],"varies":[83],"per":[84],"sample,":[85],"closed-form":[87],"dependence":[88],"on":[89,110],"advantage.":[95],"identity":[97],"holds":[98],"at":[99,145],"every":[100],"minibatch":[101],"step":[102,143],"across":[104],"entire":[106],"inner":[107],"loop,":[108],"five":[111],"MuJoCo":[112],"continuous-control":[113],"benchmarks":[114],"losses":[117],"produce":[118],"indistinguishable":[119],"training":[120],"curves.":[121],"reformulation":[123],"exposes":[124],"structural":[126],"feature":[127],"min":[134],"notation":[135],"hides.":[136],"PPO-Clip's":[137],"implicit":[138],"per-sample":[139],"function":[144],"boundary":[147],"trust":[150],"region,":[151],"shape":[154],"this":[156],"natural":[160],"design":[161],"axis":[162],"generalising":[164],"algorithm.":[166],"sketch":[168],"resulting":[170],"follow-up":[171],"directions":[172],"discussion.":[175]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-25T00:00:00"}