{"id":"https://openalex.org/W7165800724","doi":"https://doi.org/10.48550/arxiv.2606.24143","title":"AsyncOPD: How Stale Can On-Policy Distillation Be?","display_name":"AsyncOPD: How Stale Can On-Policy Distillation Be?","publication_year":2026,"publication_date":"2026-06-23","ids":{"openalex":"https://openalex.org/W7165800724","doi":"https://doi.org/10.48550/arxiv.2606.24143"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.24143","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.24143","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.24143","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000868881","display_name":"Wonjun Kang","orcid":"https://orcid.org/0000-0001-5369-6011"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kang, Wonjun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139234612","display_name":"Kevin Galim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Galim, Kevin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107894055","display_name":"S H Oh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Oh, Seunghyuk","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126417395","display_name":"Minjun Kang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kang, Minjun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139244685","display_name":"Sanghyun Park","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Park, Sanghyun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139222357","display_name":"Donghoon Kim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Donghoon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139222984","display_name":"Minjae Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Minjae","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139248210","display_name":"Minseo Kim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Minseo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139268703","display_name":"Rishabh Tiwari","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tiwari, Rishabh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139292108","display_name":"Yuchen Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Yuchen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139292680","display_name":"Hyung Il Koo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Koo, Hyung Il","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139233162","display_name":"Kangwook Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Kangwook","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.20389999449253082,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.20389999449253082,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.09189999848604202,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.08869999647140503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/asynchronous-communication","display_name":"Asynchronous communication","score":0.8396000266075134},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.7825999855995178},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6388999819755554},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5235999822616577},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.4661000072956085},{"id":"https://openalex.org/keywords/decoupling","display_name":"Decoupling (probability)","score":0.4075999855995178},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.3723999857902527},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.364300012588501}],"concepts":[{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.8396000266075134},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7833999991416931},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.7825999855995178},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6388999819755554},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5235999822616577},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.4661000072956085},{"id":"https://openalex.org/C205606062","wikidata":"https://www.wikidata.org/wiki/Q5249645","display_name":"Decoupling (probability)","level":2,"score":0.4075999855995178},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.3723999857902527},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.364300012588501},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.3598000109195709},{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.35359999537467957},{"id":"https://openalex.org/C19499675","wikidata":"https://www.wikidata.org/wiki/Q232207","display_name":"Monte Carlo method","level":2,"score":0.3465999960899353},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.32519999146461487},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.30970001220703125},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.301800012588501},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2973000109195709},{"id":"https://openalex.org/C99138194","wikidata":"https://www.wikidata.org/wiki/Q183427","display_name":"Hash function","level":2,"score":0.2745000123977661},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.25459998846054077},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.25060001015663147}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.24143","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.24143","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.24143","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.24143","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7919884920120239}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"On-policy":[0],"distillation":[1],"(OPD)":[2],"trains":[3],"a":[4,94,173,196,226],"student":[5,184],"on":[6,93],"its":[7,75],"own":[8],"rollouts":[9,37],"guided":[10],"by":[11,52,244],"teacher":[12,98,108],"feedback":[13,99],"and":[14,106,201,223],"is":[15,100,134,144],"becoming":[16],"increasingly":[17],"important":[18],"for":[19,42,147,199],"large":[20],"language":[21],"model":[22],"(LLM)":[23],"post-training.":[24],"Like":[25],"reinforcement":[26],"learning":[27],"(RL),":[28],"however,":[29],"OPD":[30,78,163,204,229],"faces":[31],"an":[32],"on-policy":[33],"systems":[34],"bottleneck,":[35],"as":[36],"can":[38,48,161],"dominate":[39],"training":[40,46,230,242,251],"time":[41],"reasoning":[43],"workloads.":[44],"Asynchronous":[45],"pipelines":[47],"alleviate":[49],"this":[50,148],"bottleneck":[51],"decoupling":[53],"rollout":[54],"generation":[55],"from":[56,233],"learner":[57,186],"updates,":[58],"but":[59],"doing":[60],"so":[61],"introduces":[62],"stale-policy":[63],"data.":[64],"While":[65],"prior":[66],"work":[67],"has":[68],"studied":[69],"stale":[70,138],"data":[71],"in":[72,77,89],"asynchronous":[73,90,159,228],"RL,":[74],"effects":[76],"remain":[79],"underexplored.":[80],"We":[81,121],"present":[82,222],"the":[83,128,178,182],"first":[84,122],"systematic":[85],"study":[86,153],"of":[87],"staleness":[88],"OPD,":[91],"focusing":[92],"practical":[95],"setting":[96],"where":[97],"implemented":[101],"through":[102],"local":[103],"KL":[104,125,133,143],"losses":[105],"full-vocabulary":[107],"logits":[109],"are":[110],"too":[111],"expensive":[112],"to":[113,137,157,246],"store":[114],"or":[115],"transfer,":[116],"necessitating":[117],"finite":[118,192],"teacher-score":[119,193],"caches.":[120],"show":[123,238],"that":[124,239],"direction":[126],"changes":[127],"stale-data":[129],"problem:":[130],"teacher-weighted":[131],"forward":[132],"more":[135],"robust":[136],"rollouts,":[139],"whereas":[140],"student-weighted":[141],"reverse":[142],"vulnerable.":[145],"Second,":[146],"vulnerable":[149],"reverse-KL":[150,179,203],"case,":[151],"we":[152,189,221],"whether":[154],"methods":[155],"designed":[156],"stabilize":[158],"RL":[160],"mitigate":[162],"staleness.":[164],"In":[165],"our":[166],"experiments,":[167],"they":[168],"do":[169],"not":[170],"improve":[171],"over":[172,248],"simpler":[174],"OPD-specific":[175],"surrogate:":[176],"recomputing":[177],"signal":[180],"under":[181],"current":[183],"at":[185],"time.":[187],"Third,":[188],"analyze":[190],"how":[191],"caches":[194],"create":[195],"bias-variance":[197],"tradeoff":[198],"sparse":[200],"sampled":[202],"estimators.":[205],"This":[206],"motivates":[207],"multi-sample":[208],"Monte":[209],"Carlo":[210],"(MC),":[211],"which":[212],"preserves":[213],"MC":[214],"correctability":[215],"while":[216,252],"reducing":[217],"one-sample":[218],"variance.":[219],"Finally,":[220],"open-source":[224],"AsyncOPD,":[225],"fully":[227],"pipeline":[231],"built":[232],"these":[234],"estimator":[235],"choices.":[236],"Experiments":[237],"AsyncOPD":[240],"improves":[241],"throughput":[243],"$1.6\\times$":[245],"$3.8\\times$":[247],"strict":[249],"synchronous":[250],"reaching":[253],"comparable":[254],"accuracy.":[255]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-25T00:00:00"}