{"id":"https://openalex.org/W7168261753","doi":"https://doi.org/10.48550/arxiv.2607.10848","title":"Predictive Divergence Masks for LLM RL","display_name":"Predictive Divergence Masks for LLM RL","publication_year":2026,"publication_date":"2026-07-12","ids":{"openalex":"https://openalex.org/W7168261753","doi":"https://doi.org/10.48550/arxiv.2607.10848"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.10848","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.10848","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.10848","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140605040","display_name":"Xiangxin Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Xiangxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140677726","display_name":"Jiarui Yao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Jiarui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140709797","display_name":"Penghui Qi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi, Penghui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140667410","display_name":"Bowen Ping","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ping, Bowen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140700674","display_name":"Jiaqi Tang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Jiaqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140681165","display_name":"Haonan Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Haonan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5140576762","display_name":"Tianyu Pang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pang, Tianyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.18279999494552612,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.18279999494552612,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.10610000044107437,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.0885000005364418,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/divergence","display_name":"Divergence (linguistics)","score":0.8101999759674072},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.5838000178337097},{"id":"https://openalex.org/keywords/softmax-function","display_name":"Softmax function","score":0.5785999894142151},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.43070000410079956},{"id":"https://openalex.org/keywords/prior-probability","display_name":"Prior probability","score":0.36980000138282776},{"id":"https://openalex.org/keywords/proxy","display_name":"Proxy (statistics)","score":0.36970001459121704},{"id":"https://openalex.org/keywords/kullback\u2013leibler-divergence","display_name":"Kullback\u2013Leibler divergence","score":0.36469998955726624},{"id":"https://openalex.org/keywords/predictive-power","display_name":"Predictive power","score":0.36070001125335693}],"concepts":[{"id":"https://openalex.org/C207390915","wikidata":"https://www.wikidata.org/wiki/Q1230525","display_name":"Divergence (linguistics)","level":2,"score":0.8101999759674072},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.5838000178337097},{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.5785999894142151},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5669999718666077},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.446399986743927},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.43070000410079956},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.36980000138282776},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.36970001459121704},{"id":"https://openalex.org/C171752962","wikidata":"https://www.wikidata.org/wiki/Q255166","display_name":"Kullback\u2013Leibler divergence","level":2,"score":0.36469998955726624},{"id":"https://openalex.org/C2778136018","wikidata":"https://www.wikidata.org/wiki/Q10350689","display_name":"Predictive power","level":2,"score":0.36070001125335693},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.35850000381469727},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.3179999887943268},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.31279999017715454},{"id":"https://openalex.org/C139676723","wikidata":"https://www.wikidata.org/wiki/Q1193832","display_name":"Sign (mathematics)","level":2,"score":0.29829999804496765},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.29809999465942383},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.29809999465942383},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.29420000314712524},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.2621999979019165},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.25870001316070557},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.25839999318122864},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2524000108242035}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.10848","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.10848","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.10848","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.10848","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1],"for":[2,25,191],"large":[3],"language":[4],"models":[5],"(LLMs)":[6],"typically":[7],"relies":[8],"on":[9],"trust-region":[10],"masks":[11,217],"to":[12],"stabilize":[13],"off-policy":[14],"updates.":[15],"The":[16],"dominant":[17],"PPO-style":[18],"approach":[19],"uses":[20],"the":[21,34,41,51,61,74,95,120,123,127,133,140,148,153,157,183,197,204,208,211,215],"sampled-token":[22,96],"importance":[23,97],"ratio":[24,98],"two":[26,187],"criteria:":[27],"a":[28,45,70,111,178],"proximity":[29,62,128],"criterion,":[30,47],"which":[31,48,137],"asks":[32,49,138],"whether":[33,50,139],"policy":[35],"has":[36],"moved":[37],"too":[38],"far":[39],"from":[40,86,101],"behavior":[42,75],"policy,":[43],"and":[44,76,214,224],"direction":[46,81,108,199],"update":[52],"pushes":[53],"it":[54],"farther":[55],"away.":[56],"Recent":[57],"work":[58],"DPPO":[59],"improves":[60],"criterion":[63,82,109],"by":[64,152],"replacing":[65],"PPO's":[66],"ratio-based":[67,107],"test":[68],"with":[69,119,203],"probability":[71],"divergence":[72,124,135,150,209],"between":[73],"training":[77,220],"policies.":[78],"However,":[79],"its":[80],"is":[83,110,200],"still":[84],"inherited":[85],"PPO.":[87],"A":[88],"token":[89],"can":[90,115],"be":[91],"masked":[92],"only":[93,177],"when":[94],"moves":[99],"away":[100],"one.":[102],"We":[103,130],"observe":[104],"that":[105,114,125],"this":[106,167,192],"single-sample":[112],"proxy":[113],"disagree":[116],"in":[117,162,169],"sign":[118],"change":[121,206],"of":[122,182,207],"defines":[126],"criterion.":[129],"therefore":[131],"propose":[132],"predictive":[134],"mask,":[136],"next":[141],"policy-gradient":[142],"step":[143],"will":[144],"increase":[145],"or":[146],"decrease":[147],"same":[149],"used":[151,161],"trust":[154],"region.":[155],"For":[156],"discrete":[158],"softmax":[159],"policies":[160],"LLM":[163],"RL,":[164],"we":[165,185],"derive":[166],"prediction":[168],"closed":[170],"form.":[171],"Because":[172],"production":[173],"rollout":[174],"engines":[175],"expose":[176],"truncated":[179],"(top-K)":[180],"view":[181],"vocabulary,":[184],"develop":[186],"lightweight":[188],"top-$K$":[189],"estimators":[190],"prediction.":[193],"Detailed":[194],"analysis":[195],"shows":[196],"divergence-based":[198],"better":[201],"aligned":[202],"realized":[205],"than":[210],"sampled":[212],"ratio,":[213],"resulting":[216],"improve":[218],"RL":[219],"across":[221],"model":[222],"scales":[223],"precision":[225],"settings.":[226]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-15T00:00:00"}