{"id":"https://openalex.org/W7166046835","doi":"https://doi.org/10.48550/arxiv.2606.26790","title":"OPID: On-Policy Skill Distillation for Agentic Reinforcement Learning","display_name":"OPID: On-Policy Skill Distillation for Agentic Reinforcement Learning","publication_year":2026,"publication_date":"2026-06-25","ids":{"openalex":"https://openalex.org/W7166046835","doi":"https://doi.org/10.48550/arxiv.2606.26790"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.26790","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.26790","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.26790","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026769012","display_name":"Sen Yang","orcid":"https://orcid.org/0000-0003-1064-0055"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Shuo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139447406","display_name":"Jinyang Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Jinyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139429460","display_name":"Zhengxi Lu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Zhengxi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139456541","display_name":"Yuhao Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Yuhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047301622","display_name":"F Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Fan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139400367","display_name":"Lang Feng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feng, Lang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139406717","display_name":"Shuai Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Shuai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139442322","display_name":"Haoran Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Haoran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139433080","display_name":"Zheng Lian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lian, Zheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139399469","display_name":"Zhengqi Wen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wen, Zhengqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139405445","display_name":"Jianhua Tao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tao, Jianhua","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.22660000622272491,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.22660000622272491,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.16930000483989716,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.14569999277591705,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hindsight-bias","display_name":"Hindsight bias","score":0.9430000185966492},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7706999778747559},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.5957000255584717},{"id":"https://openalex.org/keywords/outcome","display_name":"Outcome (game theory)","score":0.3741999864578247},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.3727000057697296},{"id":"https://openalex.org/keywords/dreyfus-model-of-skill-acquisition","display_name":"Dreyfus model of skill acquisition","score":0.34619998931884766},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.32330000400543213}],"concepts":[{"id":"https://openalex.org/C10347200","wikidata":"https://www.wikidata.org/wiki/Q1960297","display_name":"Hindsight bias","level":2,"score":0.9430000185966492},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7706999778747559},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6495000123977661},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5957000255584717},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5468000173568726},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.48350000381469727},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.3741999864578247},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.3727000057697296},{"id":"https://openalex.org/C132758656","wikidata":"https://www.wikidata.org/wiki/Q5307365","display_name":"Dreyfus model of skill acquisition","level":2,"score":0.34619998931884766},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.32330000400543213},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.3111000061035156},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.2815000116825104},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.2750999927520752},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.27250000834465027},{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.2542000114917755},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.25}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.26790","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.26790","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.26790","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.26790","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Outcome-based":[0],"reinforcement":[1],"learning":[2],"provides":[3],"a":[4,75,163],"stable":[5],"optimization":[6],"backbone":[7],"for":[8,174],"language":[9],"agents,":[10],"but":[11],"its":[12],"sparse":[13],"trajectory-level":[14],"rewards":[15],"provide":[16],"little":[17],"guidance":[18,131],"on":[19,40,193],"which":[20,48,167],"intermediate":[21],"decisions":[22,120],"should":[23],"be":[24,55],"reinforced":[25],"or":[26,44,98],"suppressed.":[27],"On-policy":[28],"self-distillation":[29,165],"offers":[30],"dense":[31],"token-level":[32,164],"supervision,":[33],"yet":[34],"existing":[35,214],"skill-conditioned":[36],"variants":[37],"often":[38],"rely":[39],"external":[41],"skill":[42,79,135],"memories":[43],"retrieved":[45],"privileged":[46],"context,":[47],"are":[49,121],"costly":[50],"to":[51,126,146],"maintain":[52],"and":[53,123,155,196,208,213],"can":[54],"mismatched":[56],"with":[57,170],"the":[58,63,139,143,148,171,182],"state":[59],"distribution":[60],"induced":[61],"by":[62],"current":[64],"policy":[65,145,175],"in":[66],"multi-turn":[67],"interaction.":[68],"We":[69],"propose":[70],"\\textbf{OPID}":[71],"(\\textbf{O}n-\\textbf{P}olicy":[72],"Sk\\textbf{i}ll":[73],"\\textbf{D}istillation),":[74],"framework":[76],"that":[77,200],"extracts":[78],"supervision":[80],"directly":[81],"from":[82],"completed":[83],"on-policy":[84],"trajectories.":[85],"OPID":[86,177,201],"represents":[87],"trajectory":[88],"hindsight":[89,190],"as":[90,129,181],"hierarchical":[91],"skills:":[92],"episode-level":[93,127],"skills":[94,103,117,128],"capture":[95,104],"global":[96],"workflows":[97],"failure-avoidance":[99],"rules,":[100],"while":[101,186],"step-level":[102,116],"local":[105],"decision":[106],"knowledge":[107],"at":[108,221],"critical":[109,119],"timesteps.":[110],"A":[111],"critical-first":[112],"routing":[113],"mechanism":[114],"uses":[115],"when":[118],"identified":[122],"falls":[124],"back":[125],"default":[130],"otherwise.":[132],"The":[133,158],"selected":[134],"is":[136,168,219],"injected":[137],"into":[138],"interaction":[140],"history,":[141],"allowing":[142],"old":[144],"re-score":[147],"same":[149],"sampled":[150],"response":[151],"under":[152],"both":[153],"original":[154],"skill-augmented":[156],"contexts.":[157],"resulting":[159],"log-probability":[160],"shift":[161],"yields":[162],"advantage,":[166],"combined":[169],"outcome":[172],"advantage":[173],"optimization.":[176],"thus":[178],"preserves":[179],"RL":[180,212],"primary":[183],"training":[184],"objective":[185],"introducing":[187],"dense,":[188],"distribution-matched":[189],"supervision.":[191],"Experiments":[192],"ALFWorld,":[194],"WebShop":[195],"Search-based":[197],"QA":[198],"demonstrate":[199],"generally":[202],"improves":[203],"agent":[204],"performance,":[205],"sample":[206],"efficiency,":[207],"robustness":[209],"over":[210],"outcome-only":[211],"skill-distillation":[215],"baselines.":[216],"Our":[217],"code":[218],"available":[220],"https://github.com/jinyangwu/OPID/tree/main.":[222]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-27T00:00:00"}