{"id":"https://openalex.org/W7163131677","doi":"https://doi.org/10.48550/arxiv.2606.01249","title":"Trust Region On-Policy Distillation","display_name":"Trust Region On-Policy Distillation","publication_year":2026,"publication_date":"2026-05-31","ids":{"openalex":"https://openalex.org/W7163131677","doi":"https://doi.org/10.48550/arxiv.2606.01249"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.01249","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.01249","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.01249","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137707310","display_name":"Xingrun Xing","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xing, Xingrun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137699734","display_name":"Haoqing Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Haoqing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137616246","display_name":"Boyan Gao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Boyan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137650633","display_name":"Ziheng Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Ziheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137673002","display_name":"Yehui Tang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Yehui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3416999876499176,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3416999876499176,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.11190000176429749,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.08219999819993973,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/outlier","display_name":"Outlier","score":0.7110999822616577},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.5685999989509583},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5608999729156494},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.5199999809265137},{"id":"https://openalex.org/keywords/yield","display_name":"Yield (engineering)","score":0.4447000026702881},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.39160001277923584}],"concepts":[{"id":"https://openalex.org/C79337645","wikidata":"https://www.wikidata.org/wiki/Q779824","display_name":"Outlier","level":2,"score":0.7110999822616577},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6129000186920166},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.5685999989509583},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5608999729156494},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.5199999809265137},{"id":"https://openalex.org/C134121241","wikidata":"https://www.wikidata.org/wiki/Q899301","display_name":"Yield (engineering)","level":2,"score":0.4447000026702881},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.39160001277923584},{"id":"https://openalex.org/C110121322","wikidata":"https://www.wikidata.org/wiki/Q865811","display_name":"Distribution (mathematics)","level":2,"score":0.3792000114917755},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.36820000410079956},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33820000290870667},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33070001006126404},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.32600000500679016},{"id":"https://openalex.org/C141603448","wikidata":"https://www.wikidata.org/wiki/Q134830","display_name":"Prefix","level":2,"score":0.3244999945163727},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.28760001063346863},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.2734000086784363},{"id":"https://openalex.org/C151233233","wikidata":"https://www.wikidata.org/wiki/Q47574","display_name":"Units of measurement","level":2,"score":0.25519999861717224}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.01249","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.01249","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.01249","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.01249","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"On-Policy":[0,70,80],"Distillation":[1],"(OPD)":[2],"is":[3],"a":[4],"fundamental":[5],"technique":[6],"for":[7],"efficient":[8],"post-training":[9],"of":[10,98,125],"large":[11],"language":[12],"models":[13],"(LLMs),":[14],"with":[15],"broad":[16],"applications":[17],"in":[18,86],"agent":[19],"learning,":[20],"multi-task":[21],"enhancement,":[22],"and":[23,34,50,66,117,138,164,171],"model":[24],"compression.":[25],"However,":[26],"OPD":[27,84,159],"training":[28],"becomes":[29],"unstable":[30],"when":[31],"the":[32,75,89,95,99,122],"teacher":[33,40,90,136],"student":[35,132],"distributions":[36],"differ":[37],"substantially,":[38],"as":[39],"supervision":[41,61],"on":[42],"student-generated":[43],"tokens":[44],"may":[45],"yield":[46],"unreliable":[47,126],"policy":[48],"gradients":[49],"even":[51],"cause":[52],"optimization":[53,96],"failure.":[54],"This":[55],"work":[56],"addresses":[57],"reliable":[58,92,150],"on-policy":[59,147],"token-level":[60],"through":[62],"credit":[63],"assignment":[64],"strategies,":[65],"proposes":[67],"Trust":[68],"Region":[69],"Distillation,":[71],"TrOPD.":[72],"It":[73],"features":[74],"following":[76],"characteristics:":[77],"1)":[78],"Trust-Region":[79],"Learning:":[81],"TrOPD":[82,155],"performs":[83],"only":[85],"regions":[87],"where":[88],"provides":[91],"supervision,":[93],"mitigating":[94],"difficulty":[97],"K1":[100],"reverse-KL":[101],"estimator":[102],"under":[103],"distribution":[104],"mismatch.":[105],"2)":[106],"Outlier":[107],"Estimation:":[108],"For":[109],"outlier":[110],"regions,":[111],"we":[112],"explore":[113],"gradient":[114],"clipping,":[115],"masking,":[116],"forward-KL":[118],"estimation":[119],"to":[120,142],"reduce":[121],"adverse":[123],"effects":[124],"supervision.":[127],"3)":[128],"Off-Policy":[129],"Guidance:":[130],"The":[131],"continues":[133],"generation":[134],"from":[135],"prefixes":[137],"uses":[139],"forward":[140],"KL":[141],"imitate":[143],"off-policy":[144],"guidance,":[145],"encouraging":[146],"exploration":[148],"toward":[149],"regions.":[151],"Experiments":[152],"show":[153],"that":[154],"consistently":[156],"outperforms":[157],"SoTA":[158],"baselines,":[160],"including":[161],"OPD,":[162],"EOPD,":[163],"REOPOLD,":[165],"across":[166],"mathematical":[167],"reasoning,":[168],"code":[169],"generation,":[170],"general-domain":[172],"benchmarks.":[173]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-03T00:00:00"}