{"id":"https://openalex.org/W7162115056","doi":"https://doi.org/10.48550/arxiv.2605.22156","title":"One-Way Policy Optimization for Self-Evolving LLMs","display_name":"One-Way Policy Optimization for Self-Evolving LLMs","publication_year":2026,"publication_date":"2026-05-21","ids":{"openalex":"https://openalex.org/W7162115056","doi":"https://doi.org/10.48550/arxiv.2605.22156"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.22156","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.22156","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.22156","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136752948","display_name":"Shuo Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Shuo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136811932","display_name":"Jinda Lu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Jinda","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136735864","display_name":"Kexin Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Kexin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100934186","display_name":"Chiyu Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Chiyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136781009","display_name":"Shaohang Wei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Shaohang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136754162","display_name":"Yuyang Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yuyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136767304","display_name":"Guoyin Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Guoyin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136785988","display_name":"Jingren Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Jingren","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136776451","display_name":"Li Yuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan, Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3546999990940094,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3546999990940094,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.20810000598430634,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.054099999368190765,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.6035000085830688},{"id":"https://openalex.org/keywords/verifiable-secret-sharing","display_name":"Verifiable secret sharing","score":0.4465000033378601},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.4318000078201294},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.39910000562667847},{"id":"https://openalex.org/keywords/decoupling","display_name":"Decoupling (probability)","score":0.374099999666214},{"id":"https://openalex.org/keywords/intersection","display_name":"Intersection (aeronautics)","score":0.3668999969959259},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.3625999987125397},{"id":"https://openalex.org/keywords/optimization-problem","display_name":"Optimization problem","score":0.35530000925064087}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6377000212669373},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.6035000085830688},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.47029998898506165},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.4465000033378601},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.4318000078201294},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.39910000562667847},{"id":"https://openalex.org/C205606062","wikidata":"https://www.wikidata.org/wiki/Q5249645","display_name":"Decoupling (probability)","level":2,"score":0.374099999666214},{"id":"https://openalex.org/C64543145","wikidata":"https://www.wikidata.org/wiki/Q162942","display_name":"Intersection (aeronautics)","level":2,"score":0.3668999969959259},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.3625999987125397},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.35530000925064087},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.34139999747276306},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.3337000012397766},{"id":"https://openalex.org/C2778869765","wikidata":"https://www.wikidata.org/wiki/Q6028363","display_name":"Inefficiency","level":2,"score":0.3264999985694885},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.29319998621940613},{"id":"https://openalex.org/C55660270","wikidata":"https://www.wikidata.org/wiki/Q5164377","display_name":"Constrained optimization","level":2,"score":0.290800005197525},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.2856000065803528},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.27320000529289246},{"id":"https://openalex.org/C2778069335","wikidata":"https://www.wikidata.org/wiki/Q32098","display_name":"Sanctions","level":2,"score":0.26489999890327454},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.25940001010894775},{"id":"https://openalex.org/C2778049539","wikidata":"https://www.wikidata.org/wiki/Q17002908","display_name":"Bayesian optimization","level":2,"score":0.25600001215934753},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.25279998779296875}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.22156","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.22156","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.22156","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.22156","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities","score":0.5870440602302551}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"Learning":[1],"with":[2],"Verifiable":[3],"Rewards":[4],"(RLVR)":[5],"has":[6],"become":[7],"a":[8,46,82,152],"promising":[9],"paradigm":[10],"for":[11,122,135],"scaling":[12],"reasoning":[13],"capabilities":[14],"of":[15,23,88,175],"Large":[16],"Language":[17],"Models":[18],"(LLMs).":[19],"However,":[20],"the":[21,63,68,86,97,100,104,111,126,130,139,142,173],"sparsity":[22],"binary":[24],"verifier":[25,98],"rewards":[26],"often":[27],"leads":[28],"to":[29,45,66,109,178],"low":[30],"efficiency":[31],"and":[32,132,170],"optimization":[33,90],"instability.":[34],"To":[35,73],"stabilize":[36],"training,":[37],"existing":[38],"methods":[39],"typically":[40],"impose":[41],"token-level":[42],"constraints":[43,53],"relative":[44],"reference":[47,105,148,186],"policy.":[48],"We":[49],"identify":[50],"that":[51,155,162],"such":[52],"penalize":[54],"deviations":[55,124,137],"indiscriminately;":[56],"this":[57],"can":[58],"flip":[59],"verifier-determined":[60],"direction":[61,91],"when":[62],"policy":[64,106,127,140],"attempts":[65],"outperform":[67],"reference,":[69],"thereby":[70],"suppressing":[71],"gains.":[72,158],"resolve":[74],"this,":[75],"we":[76],"propose":[77],"One-Way":[78],"Policy":[79],"Optimization":[80],"(OWPO),":[81],"method":[83],"based":[84],"on":[85,184],"principle":[87],"decoupling":[89],"from":[92],"update":[93,101],"magnitude.":[94,112],"In":[95],"OWPO,":[96],"dictates":[99],"direction,":[102],"while":[103],"serves":[107],"only":[108],"adjust":[110],"Specifically,":[113],"OWPO":[114,150,163],"applies":[115],"asymmetric":[116],"reweighting:":[117],"it":[118],"performs":[119],"Accelerated":[120],"Alignment":[121],"inferior":[123],"(where":[125,138],"lags":[128],"behind":[129],"reference)":[131],"Gain":[133],"Locking":[134],"superior":[136],"surpasses":[141],"reference).":[143],"Furthermore,":[144],"by":[145],"incorporating":[146],"iterative":[147],"updates,":[149],"creates":[151],"``Ratchet":[153],"Effect''":[154],"continuously":[156],"consolidates":[157],"Experimental":[159],"results":[160],"demonstrate":[161],"outperforms":[164],"strong":[165],"baselines,":[166],"including":[167],"DAPO,":[168],"OPD,":[169],"MOPD,":[171],"breaking":[172],"bottleneck":[174],"fixed":[176],"priors":[177],"enable":[179],"continuous":[180],"self-evolution":[181],"without":[182],"reliance":[183],"external":[185],"models.":[187]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-23T00:00:00"}