{"id":"https://openalex.org/W7160947271","doi":"https://doi.org/10.48550/arxiv.2605.09725","title":"On-Policy Distillation with Best-of-N Teacher Rollout Selection","display_name":"On-Policy Distillation with Best-of-N Teacher Rollout Selection","publication_year":2026,"publication_date":"2026-05-10","ids":{"openalex":"https://openalex.org/W7160947271","doi":"https://doi.org/10.48550/arxiv.2605.09725"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.09725","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09725","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.09725","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135941279","display_name":"Ke Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Ke","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135949104","display_name":"Yunjie Tian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tian, Yunjie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135968803","display_name":"DongDi Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Dongdi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135971384","display_name":"Yijiang Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yijiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104339419","display_name":"Yuanye Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yuanye","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004716468","display_name":"Vishal M. Patel","orcid":"https://orcid.org/0000-0002-5239-692X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Patel, Vishal M","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135956652","display_name":"Di Fu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Di","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.31700000166893005,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.31700000166893005,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.2093999981880188,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.061400000005960464,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.7264999747276306},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.6546000242233276},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.5888000130653381},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5105000138282776},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.46219998598098755},{"id":"https://openalex.org/keywords/forgetting","display_name":"Forgetting","score":0.42879998683929443},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.41029998660087585}],"concepts":[{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.7264999747276306},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.6546000242233276},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6373000144958496},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.5888000130653381},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5105000138282776},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.46219998598098755},{"id":"https://openalex.org/C7149132","wikidata":"https://www.wikidata.org/wiki/Q1377840","display_name":"Forgetting","level":2,"score":0.42879998683929443},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.41029998660087585},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.3375999927520752},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.3337000012397766},{"id":"https://openalex.org/C173018170","wikidata":"https://www.wikidata.org/wiki/Q165678","display_name":"Microcontroller","level":2,"score":0.30809998512268066},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.30489999055862427},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.304500013589859},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2906000018119812},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.2743000090122223},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.2718999981880188},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.27059999108314514},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.260699987411499}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.09725","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09725","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.09725","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09725","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.5335837602615356,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"On-policy":[0],"distillation":[1],"(OPD),":[2],"which":[3],"supervises":[4],"a":[5,15,55,63,95,110,131,143,180,186],"student":[6,149],"on":[7,54,175,209,214,229,237],"its":[8],"own":[9],"sampled":[10,72,126],"trajectories,":[11],"has":[12],"emerged":[13],"as":[14],"data-efficient":[16],"post-training":[17],"method":[18],"for":[19,101],"improving":[20],"reasoning":[21,86,231],"while":[22],"avoiding":[23],"the":[24,31,65,71,83,116,124,139,161,166,201,210,234],"reward":[25],"dependence":[26],"of":[27,134],"reinforcement":[28],"learning":[29],"and":[30,51,137,219],"catastrophic":[32],"forgetting":[33],"often":[34,52],"observed":[35],"in":[36],"standard":[37,41,106,227],"supervised":[38],"fine-tuning.":[39],"However,":[40],"OPD":[42,108,202,228],"typically":[43],"computes":[44],"teacher":[45,58,73,118,127,135,155,172,211],"supervision":[46,66,112,199],"under":[47],"noisy":[48],"student-generated":[49],"contexts":[50],"relies":[53],"single":[56],"stochastic":[57],"rollout":[59],"per":[60],"prompt.":[61],"As":[62],"result,":[64],"signal":[67],"can":[68,75],"be":[69,76],"high-variance:":[70],"trajectory":[74,141,191],"incorrect,":[77],"uninformative,":[78],"or":[79],"poorly":[80],"matched":[81],"to":[82,184,195],"student's":[84,167],"current":[85,168],"behavior.":[87],"To":[88],"address":[89],"this":[90],"limitation,":[91],"we":[92],"propose":[93],"BRTS,":[94],"Best-of-N":[96],"Rollout":[97],"Teacher":[98],"Selection":[99],"framework":[100],"on-policy":[102],"distillation.":[103],"BRTS":[104,129,159,224],"augments":[105],"student-context":[107],"with":[109,165,205,233],"teacher-context":[111,198],"branch":[113],"constructed":[114],"from":[115,123],"curated":[117],"trajectory.":[119,212],"Rather":[120],"than":[121],"distilling":[122],"first":[125],"rollout,":[128],"samples":[130,173],"small":[132],"pool":[133],"trajectories":[136,156],"selects":[138],"auxiliary":[140,207],"using":[142],"simple":[144],"priority":[145],"rule:":[146],"correctness":[147],"first,":[148],"alignment":[150],"second.":[151],"When":[152],"multiple":[153],"correct":[154],"are":[157],"available,":[158],"chooses":[160],"one":[162],"most":[163],"aligned":[164],"behavior;":[169],"when":[170],"unconditioned":[171],"fail":[174],"harder":[176,238],"prompts,":[177],"it":[178],"invokes":[179],"ground-truth-conditioned":[181],"recovery":[182],"step":[183],"elicit":[185],"natural":[187],"derivation.":[188],"The":[189],"selected":[190],"is":[192,242],"then":[193],"used":[194],"provide":[196],"reliable":[197],"inside":[200],"loop,":[203],"augmented":[204],"an":[206],"loss":[208],"Experiments":[213],"AIME":[215,217],"2024,":[216],"2025,":[218],"AMC":[220],"2023":[221],"show":[222],"that":[223],"improves":[225],"over":[226],"challenging":[230],"benchmarks,":[232],"largest":[235],"gains":[236],"datasets.":[239],"Our":[240],"code":[241],"available":[243],"at":[244],"https://github.com/BWGZK-keke/BRTS.":[245]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-13T00:00:00"}