{"id":"https://openalex.org/W7169504124","doi":"https://doi.org/10.48550/arxiv.2607.15161","title":"On-Policy Delta Distillation","display_name":"On-Policy Delta Distillation","publication_year":2026,"publication_date":"2026-07-16","ids":{"openalex":"https://openalex.org/W7169504124","doi":"https://doi.org/10.48550/arxiv.2607.15161"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.15161","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.15161","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.15161","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5141090178","display_name":"Byeongho Heo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Heo, Byeongho","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141097560","display_name":"Jaehui Hwang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hwang, Jaehui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141087851","display_name":"Sangdoo Yun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yun, Sangdoo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5141107562","display_name":"Dongyoon Han","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Dongyoon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.6748999953269958,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.6748999953269958,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.05090000107884407,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.04690000042319298,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.8870999813079834},{"id":"https://openalex.org/keywords/signal","display_name":"SIGNAL (programming language)","score":0.45080000162124634},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.40400001406669617},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.3528999984264374},{"id":"https://openalex.org/keywords/delta","display_name":"Delta","score":0.3330000042915344},{"id":"https://openalex.org/keywords/fractionating-column","display_name":"Fractionating column","score":0.33250001072883606},{"id":"https://openalex.org/keywords/base","display_name":"Base (topology)","score":0.321399986743927}],"concepts":[{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.8870999813079834},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5257999897003174},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.45080000162124634},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.40400001406669617},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.3528999984264374},{"id":"https://openalex.org/C21880701","wikidata":"https://www.wikidata.org/wiki/Q2144042","display_name":"Process engineering","level":1,"score":0.34450000524520874},{"id":"https://openalex.org/C5072461","wikidata":"https://www.wikidata.org/wiki/Q49506","display_name":"Delta","level":2,"score":0.3330000042915344},{"id":"https://openalex.org/C154030694","wikidata":"https://www.wikidata.org/wiki/Q1436074","display_name":"Fractionating column","level":3,"score":0.33250001072883606},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3314000070095062},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.321399986743927},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.29350000619888306},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2721000015735626},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2718999981880188},{"id":"https://openalex.org/C20162079","wikidata":"https://www.wikidata.org/wiki/Q1151406","display_name":"Case-based reasoning","level":2,"score":0.2590999901294708},{"id":"https://openalex.org/C131548973","wikidata":"https://www.wikidata.org/wiki/Q593181","display_name":"Azeotropic distillation","level":3,"score":0.2547999918460846},{"id":"https://openalex.org/C133199616","wikidata":"https://www.wikidata.org/wiki/Q25386885","display_name":"Empirical modelling","level":2,"score":0.25429999828338623},{"id":"https://openalex.org/C19924549","wikidata":"https://www.wikidata.org/wiki/Q1385210","display_name":"Extractive distillation","level":3,"score":0.25189998745918274}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.15161","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.15161","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.15161","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.15161","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.5305683612823486,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"On-policy":[0],"distillation":[1,28,49,118,124],"is":[2,66],"an":[3],"alternative":[4],"post-training":[5,157],"method":[6,125],"in":[7],"reinforcement":[8],"learning":[9],"that":[10,111,139],"alleviates":[11],"the":[12,52,59,69,72,89,112,122],"constraints":[13],"imposed":[14],"by":[15,18,92],"reward":[16],"models":[17],"providing":[19],"token-level":[20],"supervision":[21],"from":[22],"a":[23,47,97,155],"teacher":[24,73],"model.":[25],"Although":[26],"on-policy":[27,117,144],"has":[29],"been":[30],"studied":[31],"and":[32,75,95,119,135],"applied":[33],"across":[34,132],"various":[35],"settings,":[36],"its":[37,76],"fundamental":[38],"design":[39],"remains":[40],"underexplored.":[41],"In":[42],"this":[43],"paper,":[44],"we":[45,109],"introduce":[46],"new":[48,123],"reward,":[50],"termed":[51],"delta":[53,64,113],"signal,":[54],"instead":[55],"of":[56],"directly":[57],"imitating":[58],"teacher's":[60],"output":[61],"distribution.":[62],"The":[63],"signal":[65,100,114],"defined":[67],"as":[68,126],"difference":[70],"between":[71],"model":[74,78],"base":[77],"prior":[79],"to":[80,121,149],"instruction":[81],"tuning":[82,94],"for":[83,101],"reasoning":[84,93,103,147],"capability.":[85],"It":[86],"therefore":[87],"captures":[88],"changes":[90],"induced":[91],"provides":[96],"more":[98],"direct":[99],"transferring":[102],"capabilities.":[104],"Using":[105],"extensive":[106],"empirical":[107],"evidence,":[108],"show":[110],"substantially":[115],"improves":[116],"refer":[120],"On-Policy":[127],"Delta":[128],"Distillation":[129],"(OPD$^2$).":[130],"Experiments":[131],"mathematics,":[133],"science,":[134],"code-reasoning":[136],"benchmarks":[137],"demonstrate":[138],"OPD$^2$":[140],"consistently":[141],"outperforms":[142],"conventional":[143],"distillation,":[145],"enabling":[146],"LLMs":[148],"achieve":[150],"strong":[151],"performance":[152],"with":[153],"only":[154],"short":[156],"period.":[158],"Code":[159],"will":[160],"be":[161],"available":[162],"at":[163],"https://github.com/naver-ai/opd2":[164]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-18T00:00:00"}