{"id":"https://openalex.org/W7214788515","doi":"https://doi.org/10.48550/arxiv.2609.35319","title":"Teacher-Student Gaps Are Not Enough: Outcome-Guided On-Policy Distillation for Multi-Turn Autonomous Agents","display_name":"Teacher-Student Gaps Are Not Enough: Outcome-Guided On-Policy Distillation for Multi-Turn Autonomous Agents","publication_year":2026,"publication_date":"2026-09-28","ids":{"openalex":"https://openalex.org/W7214788515","doi":"https://doi.org/10.48550/arxiv.2609.35319"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.35319","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35319","pdf_url":"https://arxiv.org/pdf/2609.35319","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.35319","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5153458542","display_name":"Tong Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhang, Tong","raw_affiliation_strings":["Peking University ,","Qwen Large Model Application Team , Alibaba ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University ,","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Qwen Large Model Application Team , Alibaba ,","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153538723","display_name":"Zhou Liu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Zhou","raw_affiliation_strings":["Peking University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University ,","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153445272","display_name":"Yihao Liu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Yihao","raw_affiliation_strings":["Qwen Large Model Application Team , Alibaba ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Qwen Large Model Application Team , Alibaba ,","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109676444","display_name":"Jiahua Bao","orcid":"https://orcid.org/0000-0003-0610-4321","observed_orcids":["https://orcid.org/0000-0003-0610-4321"]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bao, Jiahua","raw_affiliation_strings":["Qwen Large Model Application Team , Alibaba ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Qwen Large Model Application Team , Alibaba ,","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153409087","display_name":"Xuchen Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li, Xuchen","raw_affiliation_strings":["University of Chinese Academy of Sciences ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Chinese Academy of Sciences ,","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153410426","display_name":"Honglin Lin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lin, Honglin","raw_affiliation_strings":["Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137096715","display_name":"Tao Cheng","orcid":"https://orcid.org/0000-0002-5503-9813","observed_orcids":["https://orcid.org/0000-0002-5503-9813"]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cheng, Tao","raw_affiliation_strings":["Qwen Large Model Application Team , Alibaba ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Qwen Large Model Application Team , Alibaba ,","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101490809","display_name":"Zhihan Yu","orcid":"https://orcid.org/0009-0000-6411-9420","observed_orcids":["https://orcid.org/0009-0000-6411-9420"]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu, Zhihan","raw_affiliation_strings":["Qwen Large Model Application Team , Alibaba ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Qwen Large Model Application Team , Alibaba ,","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137868409","display_name":"Kai Tang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tang, Kai","raw_affiliation_strings":["Qwen Large Model Application Team , Alibaba ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Qwen Large Model Application Team , Alibaba ,","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121672896","display_name":"Xiaoxi Jiang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiang, Xiaoxi","raw_affiliation_strings":["Qwen Large Model Application Team , Alibaba ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Qwen Large Model Application Team , Alibaba ,","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5136972120","display_name":"Guanjun Jiang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiang, Guanjun","raw_affiliation_strings":["Qwen Large Model Application Team , Alibaba ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Qwen Large Model Application Team , Alibaba ,","institution_ids":["https://openalex.org/I45928872"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9761000275611877,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9761000275611877,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.00860000029206276,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.0034000000450760126,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/on-policy-distillation","display_name":"on-policy distillation","score":0.9990000128746033},{"id":"https://openalex.org/keywords/llm-based-agents","display_name":"LLM agents","score":0.5379999876022339},{"id":"https://openalex.org/keywords/alfworld","display_name":"ALFWorld","score":0.6380000114440918},{"id":"https://openalex.org/keywords/webshop","display_name":"WebShop","score":0.9819999933242798}],"concepts":[{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7221999764442444},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.642799973487854},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.5630000233650208},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.546500027179718},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.505299985408783},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.49889999628067017},{"id":"https://openalex.org/C190839683","wikidata":"https://www.wikidata.org/wiki/Q2448197","display_name":"Train","level":2,"score":0.46970000863075256},{"id":"https://openalex.org/C148043351","wikidata":"https://www.wikidata.org/wiki/Q4456944","display_name":"Current (fluid)","level":2,"score":0.4219000041484833},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.392300009727478},{"id":"https://openalex.org/C2780665704","wikidata":"https://www.wikidata.org/wiki/Q959298","display_name":"Intervention (counseling)","level":2,"score":0.3384000062942505},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.3172000050544739},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.31150001287460327},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2980000078678131},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.29789999127388},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.28519999980926514},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.26499998569488525}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.35319","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35319","pdf_url":"https://arxiv.org/pdf/2609.35319","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.35319","doi":"https://doi.org/10.48550/arxiv.2609.35319","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.35319","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.35319","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35319","pdf_url":"https://arxiv.org/pdf/2609.35319","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.9235000014305115,"display_name":"Quality education","id":"https://metadata.un.org/sdg/4"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214788515.pdf","grobid_xml":"https://content.openalex.org/works/W7214788515.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7172400709","https://openalex.org/W7223059966","https://openalex.org/W7214931676","https://openalex.org/W7164915140","https://openalex.org/W7214711013","https://openalex.org/W7223260451","https://openalex.org/W7214777665","https://openalex.org/W7204908525","https://openalex.org/W7157298760","https://openalex.org/W7214728041"],"abstract_inverted_index":{"On-policy":[0],"distillation":[1],"(OPD)":[2],"trains":[3],"a":[4,37,47,101,107],"student":[5,80,87,112,142,174],"on":[6,16,76,181],"its":[7],"own":[8],"trajectories":[9,185],"with":[10,82],"dense":[11],"teacher":[12,73,127,162,189],"supervision.":[13],"Recent":[14],"work":[15],"OPD":[17,218],"for":[18,40],"multi-turn":[19],"autonomous":[20],"agents":[21],"often":[22],"treats":[23],"large":[24,50],"teacher-student":[25],"token-level":[26],"distributional":[27],"gaps":[28,35,51,57,62,118],"as":[29],"promising":[30],"intervention":[31],"points,":[32],"linking":[33],"larger":[34],"to":[36,106,124,161,222],"greater":[38],"need":[39],"correction.":[41],"Yet,":[42],"our":[43],"empirical":[44],"analysis":[45],"reveals":[46],"supervision-benefit":[48],"mismatch:":[49],"can":[52,58,143],"be":[53,59],"benign,":[54],"while":[55,100],"small":[56],"outcome-critical.":[60],"Teacher-student":[61],"capture":[63],"differences":[64],"at":[65,186],"the":[66,70,78,83,98,111,115,130,140,182,192,227],"current":[67,79,131,141,193],"turn,":[68],"whereas":[69],"benefit":[71],"of":[72],"guidance":[74,128,138,190],"depends":[75],"how":[77],"interacts":[81],"environment":[84],"afterward.":[85],"The":[86],"may":[88,104],"still":[89],"succeed":[90],"despite":[91],"choosing":[92],"an":[93],"action":[94,103],"that":[95,139],"differs":[96],"from":[97,109,172],"teacher's,":[99],"teacher-preferred":[102],"lead":[105],"state":[108],"which":[110,157],"cannot":[113],"complete":[114],"task.":[116],"Local":[117],"alone":[119],"are":[120],"therefore":[121],"not":[122],"enough":[123],"determine":[125],"whether":[126],"benefits":[129,191],"student.":[132,194],"Effective":[133],"supervision":[134,163,180],"should":[135],"instead":[136],"emphasize":[137],"translate":[144],"into":[145],"better":[146],"final":[147,169],"task":[148,170,209],"outcomes.":[149],"Accordingly,":[150],"we":[151],"propose":[152],"Outcome-Guided":[153],"On-Policy":[154],"Distillation":[155],"(OG-OPD),":[156],"applies":[158],"trajectory-relative":[159],"weighting":[160],"and":[164,198,219],"calibrates":[165],"these":[166],"weights":[167],"using":[168],"outcomes":[171],"paired":[173],"continuations.":[175],"This":[176],"calibration":[177],"selectively":[178],"strengthens":[179],"student's":[183],"original":[184],"turns":[187],"where":[188],"Across":[195],"ALFWorld,":[196],"ScienceWorld,":[197],"WebShop,":[199],"OG-OPD":[200],"consistently":[201],"outperforms":[202],"baselines":[203],"under":[204],"diverse":[205],"settings.":[206],"It":[207],"improves":[208],"success":[210],"rates":[211],"by":[212,220],"3.6-17.7":[213],"percentage":[214,224],"points":[215,225],"over":[216,226],"vanilla":[217],"up":[221],"7.0":[223],"strongest":[228],"baseline.":[229]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-09-30T00:00:00"}