{"id":"https://openalex.org/W7204222579","doi":"https://doi.org/10.48550/arxiv.2608.24696","title":"On-policy Distillation with Verifiable Reward","display_name":"On-policy Distillation with Verifiable Reward","publication_year":2026,"publication_date":"2026-08-25","ids":{"openalex":"https://openalex.org/W7204222579","doi":"https://doi.org/10.48550/arxiv.2608.24696"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.24696","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.24696","pdf_url":"https://arxiv.org/pdf/2608.24696","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.24696","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5123326411","display_name":"Wenze Lin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lin, Wenze","raw_affiliation_strings":["LeapLab , Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LeapLab , Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125660742","display_name":"Jiale Zhao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhao, Jiale","raw_affiliation_strings":["Beihang University","LeapLab , Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beihang University","institution_ids":["https://openalex.org/I82880672"]},{"raw_affiliation_string":"LeapLab , Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125206336","display_name":"Xitai Jiang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiang, Xitai","raw_affiliation_strings":["LeapLab , Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LeapLab , Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148319891","display_name":"Songde Rao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rao, Songde","raw_affiliation_strings":["SMS , Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SMS , Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148305000","display_name":"Yining Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li, Yining","raw_affiliation_strings":["LeapLab , Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LeapLab , Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123615132","display_name":"Shenzhi Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wang, Shenzhi","raw_affiliation_strings":["LeapLab , Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LeapLab , Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140196753","display_name":"Bingxiang He","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"He, Bingxiang","raw_affiliation_strings":["NLPLab , Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLPLab , Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5136799790","display_name":"Gao Huang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huang, Gao","raw_affiliation_strings":["LeapLab , Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LeapLab , Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":11,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.5404999852180481,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.5404999852180481,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2531999945640564,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.08950000256299973,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/on-policy-distillation","display_name":"on-policy distillation","score":0.9909999966621399},{"id":"https://openalex.org/keywords/verifiable-rewards","display_name":"verifiable rewards","score":0.8009999990463257},{"id":"https://openalex.org/keywords/reinforcement-learning-with-verifiable-rewards","display_name":"reinforcement learning with verifiable rewards","score":0.8880000114440918},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8050000071525574},{"id":"https://openalex.org/keywords/post-training-fine-tuning","display_name":"post-training fine-tuning","score":0.6380000114440918},{"id":"https://openalex.org/keywords/group-relative-policy-optimization","display_name":"Group Relative Policy Optimization","score":0.6150000095367432},{"id":"https://openalex.org/keywords/reasoning-benchmarks","display_name":"reasoning benchmarks","score":0.7039999961853027},{"id":"https://openalex.org/keywords/teacher-student-alignment","display_name":"teacher-student alignment","score":0.4659999907016754}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7967000007629395},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.7613999843597412},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.6499999761581421},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.6399000287055969},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.6308000087738037},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.5429999828338623},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.5230000019073486},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5145000219345093},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.4964999854564667},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.4478999972343445},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.42820000648498535},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40689998865127563},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.399399995803833},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.3937999904155731},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38109999895095825},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.31940001249313354},{"id":"https://openalex.org/C30905978","wikidata":"https://www.wikidata.org/wiki/Q815598","display_name":"Metering mode","level":2,"score":0.3066999912261963},{"id":"https://openalex.org/C138268822","wikidata":"https://www.wikidata.org/wiki/Q1051925","display_name":"Resolution (logic)","level":2,"score":0.30309998989105225},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3012000024318695},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2687000036239624},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.2597000002861023},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.2515999972820282}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.24696","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.24696","pdf_url":"https://arxiv.org/pdf/2608.24696","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.24696","doi":"https://doi.org/10.48550/arxiv.2608.24696","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.24696","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.24696","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.24696","pdf_url":"https://arxiv.org/pdf/2608.24696","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204222579.pdf","grobid_xml":"https://content.openalex.org/works/W7204222579.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W2736601468","https://openalex.org/W4391631327","https://openalex.org/W4403746689","https://openalex.org/W4405766390","https://openalex.org/W4414281281","https://openalex.org/W4414939355","https://openalex.org/W4415329135","https://openalex.org/W4416867505","https://openalex.org/W4417078041","https://openalex.org/W6929293117","https://openalex.org/W6967114859","https://openalex.org/W7083839464","https://openalex.org/W7106335775","https://openalex.org/W7118867773","https://openalex.org/W7128819059","https://openalex.org/W7130402983","https://openalex.org/W7134813685","https://openalex.org/W7135182870","https://openalex.org/W7141388625","https://openalex.org/W7148703720","https://openalex.org/W7150828934","https://openalex.org/W7154505894","https://openalex.org/W7160438937","https://openalex.org/W7160854431","https://openalex.org/W7163131677","https://openalex.org/W7165385035","https://openalex.org/W7169874691","https://openalex.org/W7170063487"],"related_works":["https://openalex.org/W7208738284","https://openalex.org/W7214931014","https://openalex.org/W7172393851","https://openalex.org/W7148703720","https://openalex.org/W7203935495","https://openalex.org/W7167615039","https://openalex.org/W7161165287","https://openalex.org/W7154206010","https://openalex.org/W7221747803","https://openalex.org/W7164043033"],"abstract_inverted_index":{"Reinforcement":[0],"Learning":[1],"with":[2,82,138,162],"Verifiable":[3,83],"Rewards":[4],"(RLVR)":[5],"and":[6,76,95,128],"on-policy":[7],"distillation":[8,136],"(OPD)":[9],"have":[10],"become":[11],"two":[12],"widely":[13],"adopted":[14],"paradigms":[15],"for":[16],"post-training":[17],"large":[18],"language":[19],"models.":[20],"However,":[21],"RLVR":[22,58,96,156],"suffers":[23],"from":[24],"sparse":[25],"task-level":[26,60],"feedback,":[27],"while":[28,57,141],"OPD":[29,52,94,109,152],"provides":[30,59],"dense":[31,54],"token-level":[32],"guidance":[33],"but":[34],"ignores":[35],"trajectory":[36,112],"correctness,":[37,113],"limiting":[38],"its":[39],"performance":[40],"to":[41,120],"that":[42,91,122,176],"of":[43,107],"the":[44,104,135,143],"teacher.":[45],"Combining":[46],"them":[47],"is":[48,184],"a":[49,86,116,154],"promising":[50],"direction:":[51],"supplies":[53],"supervisory":[55],"signals,":[56],"correctness.":[61],"Nevertheless,":[62],"existing":[63],"integrations":[64],"often":[65],"rely":[66],"on":[67,111,171],"weighted":[68],"combination":[69],"or":[70],"heuristic":[71],"switching,":[72],"introducing":[73],"extra":[74],"hyperparameters":[75],"trade-offs.":[77],"We":[78,101],"propose":[79],"On-policy":[80],"Distillation":[81],"Reward":[84],"(OPDVR),":[85],"simple":[87],"yet":[88],"effective":[89],"method":[90],"seamlessly":[92],"combines":[93],"without":[97],"adding":[98],"any":[99,163],"hyperparameters.":[100],"first":[102],"reformulate":[103],"implicit":[105],"reward":[106],"sampled-token":[108,151],"based":[110],"then":[114],"apply":[115],"ReLU":[117],"gating":[118],"mechanism":[119],"ensure":[121],"correct":[123],"trajectories":[124],"receive":[125,131],"non-negative":[126],"rewards":[127],"incorrect":[129],"ones":[130],"non-positive":[132],"rewards---thereby":[133],"aligning":[134],"signal":[137],"task":[139],"success":[140],"preserving":[142],"teacher's":[144],"distributional":[145],"guidance.":[146],"Furthermore,":[147],"our":[148],"modification":[149],"transforms":[150],"into":[153],"proper":[155],"method,":[157],"making":[158],"it":[159],"readily":[160],"combinable":[161],"policy":[164],"gradient":[165],"algorithm,":[166],"such":[167],"as":[168],"GRPO.":[169],"Experiments":[170],"six":[172],"reasoning":[173],"benchmarks":[174],"show":[175],"OPDVR":[177],"consistently":[178],"outperforms":[179],"standard":[180],"OPD.":[181],"Our":[182],"code":[183],"available":[185],"at":[186],"https://github.com/LeapLabTHU/OPDVR.":[187]},"counts_by_year":[{"year":2026,"cited_by_count":11}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-27T00:00:00"}