{"id":"https://openalex.org/W7169884067","doi":"https://doi.org/10.48550/arxiv.2607.18082","title":"CriPO: Enhancing Rubric-based RL via Self-Distillation","display_name":"CriPO: Enhancing Rubric-based RL via Self-Distillation","publication_year":2026,"publication_date":"2026-07-20","ids":{"openalex":"https://openalex.org/W7169884067","doi":"https://doi.org/10.48550/arxiv.2607.18082"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.18082","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.18082","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.18082","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5141248129","display_name":"Mingxuan Xia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xia, Mingxuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141288838","display_name":"Yuhang Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Yuhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141296189","display_name":"Chao Ye","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ye, Chao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5105989284","display_name":"S J Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Shuai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141282379","display_name":"Shenzhi Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Shenzhi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000700814","display_name":"G. Zhu","orcid":"https://orcid.org/0000-0003-4364-866X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Guangcheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141270805","display_name":"Yuhang Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yuhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141241651","display_name":"Cheng Peng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Cheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141310838","display_name":"Haobo Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Haobo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5140268496","display_name":"Siqing Wang","orcid":"https://orcid.org/0009-0009-1079-643X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Siqing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.3402000069618225,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.3402000069618225,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.1282999962568283,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.055799998342990875,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/counterfactual-thinking","display_name":"Counterfactual thinking","score":0.6470999717712402},{"id":"https://openalex.org/keywords/optimization-problem","display_name":"Optimization problem","score":0.46299999952316284},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.4406999945640564},{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.4205999970436096},{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.367000013589859},{"id":"https://openalex.org/keywords/term","display_name":"Term (time)","score":0.3391000032424927}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6690999865531921},{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.6470999717712402},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.46299999952316284},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4406999945640564},{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.4205999970436096},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.367000013589859},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.3391000032424927},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.33730000257492065},{"id":"https://openalex.org/C48677424","wikidata":"https://www.wikidata.org/wiki/Q6888088","display_name":"Mode (computer interface)","level":2,"score":0.32820001244544983},{"id":"https://openalex.org/C50817715","wikidata":"https://www.wikidata.org/wiki/Q79895177","display_name":"Regret","level":2,"score":0.29350000619888306},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.2671000063419342}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.18082","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.18082","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.18082","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.18082","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Rubric-based":[0],"RL":[1,18,166],"has":[2],"recently":[3],"shown":[4],"promise":[5],"in":[6,201],"improving":[7],"LLMs":[8],"on":[9,59,221],"open-ended":[10],"tasks.":[11],"A":[12],"widely":[13],"recognized":[14],"limitation":[15],"of":[16,130,141],"rubric-based":[17,165,231],"is":[19,57,68],"limited":[20],"exploration:":[21],"criteria":[22,96],"that":[23,89,97,123,215,227],"no":[24,33],"rollout":[25],"manages":[26],"to":[27,183,197,209],"satisfy":[28],"(Unexplored":[29],"Criteria,":[30],"UC)":[31],"receive":[32],"optimization":[34,110,241],"signal.":[35],"Recent":[36],"methods":[37],"address":[38,148],"this":[39,66,133],"by":[40,100],"incorporating":[41],"rubric":[42],"information":[43],"as":[44],"external":[45,63],"guidance":[46,64,67],"during":[47,109],"rollout,":[48],"yet":[49,103],"they":[50],"introduce":[51],"a":[52,84,174,179,194],"train-inference":[53],"mismatch:":[54],"the":[55,188],"policy":[56],"optimized":[58],"rollouts":[60,102,203],"produced":[61],"under":[62],"while":[65],"absent":[69],"at":[70],"inference":[71],"time,":[72],"causing":[73],"error":[74],"accumulation":[75],"through":[76],"autoregressive":[77],"decoding.":[78],"Moreover,":[79],"these":[80],"exploration-focused":[81],"approaches":[82],"overlook":[83],"fundamentally":[85],"different":[86],"failure":[87,134],"mode":[88,135],"we":[90,157],"term":[91],"Suppressed":[92],"Criteria":[93],"(SC)":[94],"--":[95],"are":[98,107,125],"satisfied":[99],"some":[101],"whose":[104],"learning":[105],"signals":[106],"lost":[108],"because":[111],"scalar":[112],"reward":[113],"aggregation":[114],"assigns":[115],"them":[116],"non-positive":[117],"aggregate":[118],"advantages.":[119],"Our":[120],"analysis":[121],"reveals":[122],"SC":[124,143,152],"remarkably":[126],"prevalent:":[127],"over":[128],"57%":[129],"samples":[131],"exhibit":[132],"throughout":[136],"training,":[137],"with":[138,237],"an":[139],"average":[140],"1.8":[142],"per":[144],"sample.":[145],"To":[146],"simultaneously":[147],"both":[149],"UC":[150],"and":[151,177,204,223],"without":[153],"introducing":[154],"training-inference":[155],"mismatch,":[156],"propose":[158],"Criterion-Distilled":[159],"Policy":[160],"Optimization":[161],"(CriPO),":[162],"which":[163],"enhances":[164],"via":[167],"on-policy":[168],"self-distillation.":[169],"For":[170,190],"UC,":[171],"CriPO":[172,192,228],"constructs":[173],"criterion-injection":[175],"self-teacher":[176,196],"computes":[178],"localized":[180],"forward-KL":[181],"loss":[182],"inject":[184],"missing":[185],"behaviors":[186],"into":[187],"policy.":[189],"SC,":[191],"employs":[193],"counterfactual":[195],"locate":[198],"criterion-relevant":[199],"tokens":[200],"negative-advantage":[202],"flips":[205],"their":[206],"token-level":[207],"advantages":[208],"positive":[210],"values,":[211],"preserving":[212],"useful":[213],"patterns":[214],"would":[216],"otherwise":[217],"be":[218],"suppressed.":[219],"Experiments":[220],"medicine":[222],"science":[224],"benchmarks":[225],"demonstrate":[226],"consistently":[229],"outperforms":[230],"RL,":[232],"achieving":[233],"stronger":[234],"final":[235],"performance":[236],"approximately":[238],"$2\\times$":[239],"fewer":[240],"steps.":[242]},"counts_by_year":[],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2026-07-22T00:00:00"}