{"id":"https://openalex.org/W7203904183","doi":"https://doi.org/10.48550/arxiv.2608.18682","title":"RTPO: Reverse-Turn Policy Optimization for Stabilizing Agentic RL Training","display_name":"RTPO: Reverse-Turn Policy Optimization for Stabilizing Agentic RL Training","publication_year":2026,"publication_date":"2026-08-19","ids":{"openalex":"https://openalex.org/W7203904183","doi":"https://doi.org/10.48550/arxiv.2608.18682"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.18682","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.18682","pdf_url":"https://arxiv.org/pdf/2608.18682","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.18682","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5153453478","display_name":"Yugu Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405259444","display_name":"Adelaide University","ror":"https://ror.org/028g18b61","country_code":"AU","type":"education","lineage":["https://openalex.org/I4405259444"]},{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I4405259444","https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Li, Yugu","raw_affiliation_strings":["School of CSIT Adelaide University Adelaide , SA 5000 , Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of CSIT Adelaide University Adelaide , SA 5000 , Australia","institution_ids":["https://openalex.org/I4405259444","https://openalex.org/I5681781"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030589099","display_name":"Jimmy Cao","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Zehong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043934955","display_name":"Jianglin Qiao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Qiao, Jianglin","raw_affiliation_strings":["ACFR","The University of Sydney Camperdown , NSW 2050 , Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ACFR","institution_ids":[]},{"raw_affiliation_string":"The University of Sydney Camperdown , NSW 2050 , Australia","institution_ids":["https://openalex.org/I129604602"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5147732525","display_name":"Siyi Hu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I205640436","display_name":"Curtin University","ror":"https://ror.org/02n415q13","country_code":"AU","type":"education","lineage":["https://openalex.org/I205640436"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Hu, Siyi","raw_affiliation_strings":["School of EECMS Curtin University Bentley , WA 6102 , Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of EECMS Curtin University Bentley , WA 6102 , Australia","institution_ids":["https://openalex.org/I205640436"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.0005000000237487257,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/training-stability","display_name":"training stability","score":0.6940000057220459},{"id":"https://openalex.org/keywords/llm-based-agents","display_name":"LLM agents","score":0.6919999718666077},{"id":"https://openalex.org/keywords/tool-using-agents","display_name":"tool-using agents","score":0.9039999842643738}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7894999980926514},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.7339000105857849},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.708299994468689},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.7017999887466431},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.5968000292778015},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.44020000100135803},{"id":"https://openalex.org/C88626702","wikidata":"https://www.wikidata.org/wiki/Q1128903","display_name":"Continuation","level":2,"score":0.42410001158714294},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.40779998898506165},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.40450000762939453},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.39239999651908875},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37560001015663147},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.3578999936580658},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.35519999265670776},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.3425000011920929},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3416999876499176},{"id":"https://openalex.org/C173246807","wikidata":"https://www.wikidata.org/wiki/Q7833062","display_name":"Trajectory optimization","level":3,"score":0.328000009059906},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.299699991941452},{"id":"https://openalex.org/C2779664074","wikidata":"https://www.wikidata.org/wiki/Q3518405","display_name":"Terminal (telecommunication)","level":2,"score":0.25540000200271606}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.18682","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.18682","pdf_url":"https://arxiv.org/pdf/2608.18682","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.18682","doi":"https://doi.org/10.48550/arxiv.2608.18682","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.18682","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.18682","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.18682","pdf_url":"https://arxiv.org/pdf/2608.18682","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7203904183.pdf","grobid_xml":"https://content.openalex.org/works/W7203904183.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W2736601468","https://openalex.org/W2908510526","https://openalex.org/W3210904070","https://openalex.org/W4386768656","https://openalex.org/W4388964563","https://openalex.org/W4391631327","https://openalex.org/W4402671952","https://openalex.org/W4403746689","https://openalex.org/W4403883066","https://openalex.org/W4406774573","https://openalex.org/W4406840771","https://openalex.org/W4414281281","https://openalex.org/W4414939355","https://openalex.org/W4415109242","https://openalex.org/W4415276218","https://openalex.org/W4415317052","https://openalex.org/W4415330282","https://openalex.org/W4415990083","https://openalex.org/W4416020713","https://openalex.org/W4416118517","https://openalex.org/W4416453547","https://openalex.org/W4417167303","https://openalex.org/W6891936057","https://openalex.org/W6929370958","https://openalex.org/W7119470779","https://openalex.org/W7128364308"],"related_works":["https://openalex.org/W7120077454","https://openalex.org/W7160258489","https://openalex.org/W7116953735","https://openalex.org/W7128364308","https://openalex.org/W4416777406","https://openalex.org/W7160633711","https://openalex.org/W4416146915","https://openalex.org/W4416780393","https://openalex.org/W4414810684","https://openalex.org/W7161154181"],"abstract_inverted_index":{"Training":[0],"multi-turn":[1,27,111,175],"agentic":[2,176],"workflows":[3],"with":[4,129],"reinforcement":[5],"learning":[6],"(RL)":[7],"enables":[8,134],"large":[9],"language":[10],"models":[11],"to":[12,143,170,196],"perform":[13],"complex":[14],"reasoning,":[15],"use":[16],"external":[17],"tools,":[18],"and":[19,66,72,95,117,140,157,168,185,190],"conduct":[20],"iterative":[21],"search":[22],"beyond":[23],"single-turn":[24],"settings.":[25],"Yet":[26],"RL":[28,177],"training":[29,200],"remains":[30],"highly":[31],"unstable,":[32],"often":[33],"causing":[34],"severe":[35],"performance":[36],"degradation":[37],"as":[38,113],"the":[39,161],"number":[40],"of":[41,53],"turns":[42],"increases.":[43],"Through":[44],"theoretical":[45,149],"analysis,":[46],"we":[47],"identify":[48],"three":[49],"tightly":[50],"coupled":[51],"sources":[52],"instability:":[54],"rollout-training":[55],"context":[56,155],"mismatch,":[57],"weak":[58],"turn-level":[59,119,137,163,186],"credit":[60,138,166],"assignment":[61,139],"under":[62,77,160],"sparse":[63,114],"terminal":[64],"rewards,":[65],"asynchronous":[67,145,158],"policy":[68,79,120],"drift":[69,159],"when":[70],"short":[71],"long":[73],"trajectories":[74],"are":[75],"optimized":[76],"different":[78],"versions.":[80],"We":[81,103,147],"show":[82,179],"that":[83,152,180],"these":[84],"issues":[85],"share":[86],"a":[87,99],"common":[88],"structural":[89],"origin":[90],"in":[91,122],"flattened":[92],"trajectory":[93],"optimization":[94],"address":[96],"them":[97],"through":[98],"unified":[100],"reverse-turn":[101],"formulation.":[102],"propose":[104],"Reverse-Turn":[105],"Policy":[106],"Optimization":[107],"(RTPO),":[108],"which":[109],"organizes":[110],"rollouts":[112],"reverse":[115,124],"trees":[116],"performs":[118],"updates":[121],"temporal":[123],"order,":[125],"aligning":[126],"each":[127],"decision":[128],"its":[130,194],"downstream":[131],"continuation.":[132],"RTPO":[133,153,181],"causally":[135],"consistent":[136],"on-policy":[141],"continuation":[142],"control":[144],"drift.":[146],"provide":[148],"guarantees":[150],"showing":[151],"eliminates":[154],"mismatch":[156],"proposed":[162],"formulation,":[164],"reduces":[165],"bias,":[167],"converges":[169],"recursive":[171],"optimality.":[172],"Experiments":[173],"on":[174],"benchmarks":[178],"improves":[182],"upon":[183],"trajectory-":[184],"baselines":[187],"by":[188],"21.50%":[189],"10.76%,":[191],"respectively,":[192],"highlighting":[193],"potential":[195],"support":[197],"more":[198],"stable":[199],"for":[201],"tool-using":[202],"agents.":[203]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-22T00:00:00"}