{"id":"https://openalex.org/W7161162008","doi":"https://doi.org/10.48550/arxiv.2605.12652","title":"Multi-Rollout On-Policy Distillation via Peer Successes and Failures","display_name":"Multi-Rollout On-Policy Distillation via Peer Successes and Failures","publication_year":2026,"publication_date":"2026-05-12","ids":{"openalex":"https://openalex.org/W7161162008","doi":"https://doi.org/10.48550/arxiv.2605.12652"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.12652","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12652","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.12652","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5108259382","display_name":"Weichen Yu","orcid":"https://orcid.org/0009-0003-7935-2358"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Yu, Weichen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136174359","display_name":"Xiaomin Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Xiaomin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136104055","display_name":"Yizhou Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Yizhou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063552937","display_name":"Xiaoze Liu","orcid":"https://orcid.org/0000-0002-9726-3397"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xiaoze","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126300871","display_name":"Ruowang Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Ruowang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136136526","display_name":"Haixin Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Haixin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136119705","display_name":"Yinyi Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Yinyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136112852","display_name":"Chen Henry Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Chen Henry","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136112574","display_name":"Gaurav Mittal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mittal, Gaurav","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136166320","display_name":"Matt Fredrikson","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fredrikson, Matt","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136178054","display_name":"Yu Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":11,"corresponding_author_ids":["https://openalex.org/A5108259382"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5864999890327454,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5864999890327454,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.0786999985575676,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.05480000004172325,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.7329000234603882},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.7053999900817871},{"id":"https://openalex.org/keywords/imitation","display_name":"Imitation","score":0.7020000219345093},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.6157000064849854},{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.3896999955177307},{"id":"https://openalex.org/keywords/scheme","display_name":"Scheme (mathematics)","score":0.29499998688697815}],"concepts":[{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.7329000234603882},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.7053999900817871},{"id":"https://openalex.org/C126388530","wikidata":"https://www.wikidata.org/wiki/Q1131737","display_name":"Imitation","level":2,"score":0.7020000219345093},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6815000176429749},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.6157000064849854},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.3896999955177307},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3864000141620636},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.376800000667572},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.29499998688697815},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.2847999930381775},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2782000005245209},{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.27090001106262207},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.2669000029563904},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2583000063896179}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.12652","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12652","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.12652","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12652","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7135360240936279}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"are":[3],"often":[4],"post-trained":[5],"with":[6,155],"sparse":[7],"verifier":[8,156],"rewards,":[9,157],"which":[10],"indicate":[11,170],"whether":[12],"a":[13,63],"sampled":[14,52],"trajectory":[15],"succeeds":[16,24],"but":[17],"provide":[18,92,101],"limited":[19],"guidance":[20],"about":[21,105],"where":[22],"reasoning":[23,97],"or":[25],"fails.":[26],"On-policy":[27],"distillation":[28,65,174],"(OPD)":[29],"offers":[30],"denser":[31],"token-level":[32],"supervision":[33],"by":[34],"training":[35],"on":[36,84,123],"student-generated":[37],"trajectories,":[38],"yet":[39],"existing":[40],"methods":[41],"typically":[42],"distill":[43],"each":[44],"rollout":[45,72],"independently":[46],"and":[47,87,118,131],"ignore":[48],"the":[49,54,69,82,160,177],"other":[50],"attempts":[51],"for":[53,95],"same":[55],"prompt.":[56],"We":[57,110],"introduce":[58],"Multi-Rollout":[59],"On-Policy":[60],"Distillation":[61],"(MOPD),":[62],"peer-conditioned":[64],"framework":[66],"that":[67,135,147,159,171],"uses":[68],"student's":[70,178],"local":[71],"group":[73],"to":[74,108],"construct":[75],"more":[76,164],"informative":[77],"teacher":[78,83,153],"signals.":[79],"MOPD":[80,136],"conditions":[81],"both":[85],"successful":[86],"failed":[88],"peer":[89,116],"rollouts:":[90],"successes":[91],"positive":[93,115],"evidence":[94,104],"valid":[96],"patterns,":[98],"while":[99],"failures":[100],"structured":[102],"negative":[103],"plausible":[106],"mistakes":[107],"avoid.":[109],"study":[111],"two":[112],"peer-context":[113],"constructions:":[114],"imitation":[117],"contrastive":[119],"success-failure":[120,149],"conditioning.":[121],"Experiments":[122],"competitive":[124],"programming,":[125],"mathematical":[126],"reasoning,":[127],"scientific":[128],"question":[129],"answering,":[130],"tool-use":[132],"benchmarks":[133],"show":[134],"consistently":[137],"improves":[138],"over":[139],"standard":[140],"on-policy":[141,173],"baselines.":[142],"Further":[143],"teacher-signal":[144],"analysis":[145],"shows":[146],"mixed":[148],"contexts":[150],"better":[151],"align":[152],"scores":[154],"indicating":[158],"gains":[161],"arise":[162],"from":[163],"faithful,":[165],"instance-adaptive":[166],"supervision.":[167],"These":[168],"results":[169],"effective":[172],"should":[175],"exploit":[176],"multi-rollout":[179],"trial-and-error":[180],"behavior":[181],"rather":[182],"than":[183],"treating":[184],"rollouts":[185],"as":[186],"isolated":[187],"samples.":[188]},"counts_by_year":[],"updated_date":"2026-06-03T06:16:58.514037","created_date":"2026-05-15T00:00:00"}