{"id":"https://openalex.org/W7167617612","doi":"https://doi.org/10.48550/arxiv.2607.04763","title":"Multi-Turn On-Policy Distillation with Prefix Replay","display_name":"Multi-Turn On-Policy Distillation with Prefix Replay","publication_year":2026,"publication_date":"2026-07-06","ids":{"openalex":"https://openalex.org/W7167617612","doi":"https://doi.org/10.48550/arxiv.2607.04763"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.04763","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.04763","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.04763","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140187109","display_name":"Baohao Liao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liao, Baohao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061778641","display_name":"Hanze Dong","orcid":"https://orcid.org/0000-0002-8846-1260"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong, Hanze","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140155744","display_name":"Christof Monz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Monz, Christof","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103458235","display_name":"X X Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Xinxing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140188932","display_name":"Li Dong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong, Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5140192463","display_name":"Furu Wei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Furu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.2572000026702881,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.2572000026702881,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.19519999623298645,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.10719999670982361,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.661899983882904},{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.6234999895095825},{"id":"https://openalex.org/keywords/prefix","display_name":"Prefix","score":0.5831999778747559},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5343000292778015},{"id":"https://openalex.org/keywords/schedule","display_name":"Schedule","score":0.43650001287460327},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.3944999873638153}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7803000211715698},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.661899983882904},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.6234999895095825},{"id":"https://openalex.org/C141603448","wikidata":"https://www.wikidata.org/wiki/Q134830","display_name":"Prefix","level":2,"score":0.5831999778747559},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5343000292778015},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.43650001287460327},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.3944999873638153},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.35190001130104065},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.34860000014305115},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.33739998936653137},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.33070001006126404},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3296999931335449},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.2759999930858612},{"id":"https://openalex.org/C2781020372","wikidata":"https://www.wikidata.org/wiki/Q533093","display_name":"On the fly","level":2,"score":0.27559998631477356}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.04763","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.04763","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.04763","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.04763","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.4346556067466736}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,51,86],"study":[1],"on-policy":[2],"distillation":[3,206],"(OPD)":[4],"for":[5],"agentic":[6],"tasks,":[7,209],"where":[8,111],"an":[9,14,57],"LLM":[10],"agent":[11],"interacts":[12],"with":[13,144,158],"environment":[15,44,84],"over":[16,25,163],"multiple":[17,164],"turns":[18,195],"and":[19,45,125,141,160,166,183,210],"a":[20,23,92,118,136,145,200],"student":[21,40,69,123,167,181],"imitates":[22],"teacher":[24,46,63,76,108,126,165],"these":[26],"multi-turn":[27,89,133],"interaction":[28,198],"histories.":[29,50],"Fully":[30],"online":[31],"OPD":[32,90,134],"is":[33,114,184],"costly":[34],"because":[35],"each":[36],"update":[37],"requires":[38],"fresh":[39],"rollouts":[41],"through":[42],"the":[43,68,75,102,107],"queries":[47],"at":[48,71,185],"visited":[49],"propose":[52],"Replayed-Prefix":[53],"On-Policy":[54],"Distillation":[55],"(ReOPD),":[56],"off-environment":[58],"alternative":[59],"that":[60,88,150],"reuses":[61],"pre-collected":[62],"trajectories":[64],"as":[65,135],"replayed":[66],"prefixes:":[67],"acts":[70],"selected":[72],"steps,":[73],"while":[74],"provides":[77],"dense":[78],"per-step":[79],"supervision":[80],"without":[81],"executing":[82],"new":[83],"interactions.":[85],"show":[87],"introduces":[91],"prefix":[93,138],"trap:":[94],"making":[95],"histories":[96,110],"more":[97],"student-on-policy":[98],"improves":[99,173],"relevance":[100],"to":[101],"student,":[103],"but":[104],"can":[105],"query":[106],"on":[109],"its":[112],"target":[113],"unreliable.":[115],"This":[116],"creates":[117],"two-sided":[119],"distribution":[120,139],"shift":[121],"between":[122],"occupancy":[124],"reliability.":[127],"ReOPD":[128,170,193],"addresses":[129],"this":[130],"by":[131],"treating":[132],"reliability-aware":[137],"design":[140],"implements":[142],"it":[143],"simple":[146],"step-decaying":[147],"sampling":[148],"schedule":[149],"emphasizes":[151],"early,":[152],"lower-shift":[153],"prefixes.":[154],"Across":[155],"mathematical":[156],"reasoning":[157],"Python":[159],"search":[161],"environments":[162],"model":[168],"scales,":[169],"preserves":[171],"or":[172],"OPD-level":[174],"accuracy,":[175],"uses":[176],"zero":[177],"tool":[178],"calls":[179],"during":[180],"training,":[182],"least":[186],"4$\\times$":[187],"faster":[188],"per":[189],"rollout":[190],"than":[191],"OPD.":[192],"therefore":[194],"expensive":[196],"agent-environment":[197],"into":[199],"reusable":[201],"offline":[202],"resource,":[203],"enabling":[204],"scalable":[205],"across":[207],"tools,":[208],"environments.":[211]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-08T00:00:00"}