{"id":"https://openalex.org/W7214958905","doi":"https://doi.org/10.48550/arxiv.2609.36178","title":"Targeting Pivotal Decisions for Credit Assignment in Agentic Reinforcement Learning","display_name":"Targeting Pivotal Decisions for Credit Assignment in Agentic Reinforcement Learning","publication_year":2026,"publication_date":"2026-09-28","ids":{"openalex":"https://openalex.org/W7214958905","doi":"https://doi.org/10.48550/arxiv.2609.36178"},"language":null,"primary_location":{"id":"pmh:oai:arXiv.org:2609.36178","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.36178","pdf_url":"https://arxiv.org/pdf/2609.36178","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.36178","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5153671432","display_name":"Dongwon Jung","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I84218800","display_name":"University of California, Davis","ror":"https://ror.org/05rrcem69","country_code":"US","type":"education","lineage":["https://openalex.org/I84218800"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jung, Dongwon","raw_affiliation_strings":["Microsoft","University of California , Davis"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"University of California , Davis","institution_ids":["https://openalex.org/I84218800"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036707156","display_name":"Hemanth Neelgund Ramesh","orcid":"https://orcid.org/0000-0002-6193-1592","observed_orcids":["https://orcid.org/0000-0002-6193-1592"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ramesh, Hemanth Neelgund","raw_affiliation_strings":["University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153587503","display_name":"Yifan Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Yifan","raw_affiliation_strings":["Purdue University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Purdue University","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153670700","display_name":"Xiaomin Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li, Xiaomin","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004826137","display_name":"Yuexing Hao","orcid":"https://orcid.org/0000-0003-4375-7655","observed_orcids":["https://orcid.org/0000-0003-4375-7655"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hao, Yuexing","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153586622","display_name":"Yu Hu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hu, Yu","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153669858","display_name":"Muhao Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I84218800","display_name":"University of California, Davis","ror":"https://ror.org/05rrcem69","country_code":"US","type":"education","lineage":["https://openalex.org/I84218800"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chen, Muhao","raw_affiliation_strings":["University of California , Davis"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Davis","institution_ids":["https://openalex.org/I84218800"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153617243","display_name":"Varun Chandrasekaran","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chandrasekaran, Varun","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029350772","display_name":"Andrzej Banburski","orcid":"https://orcid.org/0000-0002-3360-4121","observed_orcids":["https://orcid.org/0000-0002-3360-4121"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Banburski-Fahey, Andrzej","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5022251129","display_name":"Jaron Lanier","orcid":"https://orcid.org/0009-0002-7659-2797","observed_orcids":["https://orcid.org/0009-0002-7659-2797"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lanier, Jaron","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.0010999999940395355,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0008999999845400453,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/agentic-reinforcement-learning","display_name":"agentic reinforcement learning","score":0.9430000185966492},{"id":"https://openalex.org/keywords/credit-assignment","display_name":"credit assignment","score":0.8830000162124634},{"id":"https://openalex.org/keywords/group-relative-policy-optimization","display_name":"Group Relative Policy Optimization","score":0.8830000162124634},{"id":"https://openalex.org/keywords/llm-based-agents","display_name":"LLM agents","score":0.8270000219345093},{"id":"https://openalex.org/keywords/advantage-estimation","display_name":"advantage estimation","score":0.5820000171661377},{"id":"https://openalex.org/keywords/segment-selection","display_name":"segment selection","score":0.41100001335144043},{"id":"https://openalex.org/keywords/model-based-verification","display_name":"model-based verification","score":0.3540000021457672},{"id":"https://openalex.org/keywords/alfworld","display_name":"ALFWorld","score":0.7570000290870667},{"id":"https://openalex.org/keywords/webshop","display_name":"WebShop","score":0.9340000152587891}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6769999861717224},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5568000078201294},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.5281999707221985},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.40849998593330383},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4058000147342682},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.3578999936580658},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3564000129699707},{"id":"https://openalex.org/C159718280","wikidata":"https://www.wikidata.org/wiki/Q5526353","display_name":"Gas meter prover","level":3,"score":0.3269999921321869},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.3125999867916107},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.3077000081539154},{"id":"https://openalex.org/C29122968","wikidata":"https://www.wikidata.org/wiki/Q1414816","display_name":"Incentive","level":2,"score":0.3050999939441681},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.2896000146865845},{"id":"https://openalex.org/C2781311116","wikidata":"https://www.wikidata.org/wiki/Q83306","display_name":"Group (periodic table)","level":2,"score":0.26980000734329224},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.266400009393692},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.25859999656677246},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.2581999897956848}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.36178","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.36178","pdf_url":"https://arxiv.org/pdf/2609.36178","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.36178","doi":"https://doi.org/10.48550/arxiv.2609.36178","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.36178","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.36178","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.36178","pdf_url":"https://arxiv.org/pdf/2609.36178","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"score":0.47054916620254517,"display_name":"Peace, justice, and strong institutions","id":"https://metadata.un.org/sdg/16"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214958905.pdf","grobid_xml":"https://content.openalex.org/works/W7214958905.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Group":[0],"Relative":[1],"Policy":[2],"Optimization":[3],"(GRPO)":[4],"has":[5],"become":[6],"a":[7,46,62,75,200],"promising":[8],"approach":[9],"for":[10,53,79,178,214],"training":[11,192],"large":[12],"language":[13],"model":[14,133,167],"agents.":[15],"However,":[16],"its":[17,97],"uniform":[18],"assignment":[19,56,217],"of":[20,124,174,209],"trajectory-level":[21],"advantages":[22,123],"to":[23,28,41,73,136,139],"all":[24],"policy":[25,125,191],"tokens":[26,126],"fails":[27],"distinguish":[29],"consequential":[30],"decisions":[31,39,52,213],"from":[32,99],"less":[33],"relevant":[34],"ones,":[35],"obscuring":[36],"which":[37],"intermediate":[38,152],"contributed":[40],"success.":[42],"We":[43],"introduce":[44],"ProVer,":[45],"framework":[47],"that":[48,186],"targets":[49],"potentially":[50,77],"pivotal":[51,212],"fine-grained":[54,215],"credit":[55,144,216],"in":[57,102,145,218],"agentic":[58,66,219],"reinforcement":[59,220],"learning.":[60,221],"Given":[61],"rollout":[63],"group,":[64],"an":[65],"judge":[67,202],"contrasts":[68],"successful":[69],"and":[70,111,157,176,180,207],"failed":[71],"trajectories":[72],"propose":[74],"segment":[76,94,188],"responsible":[78],"their":[80],"divergent":[81],"outcomes.":[82],"Rather":[83],"than":[84],"directly":[85],"trusting":[86],"the":[87,92,100,113,121,128,161,205],"judge's":[88],"assessment,":[89],"ProVer":[90,141,159],"verifies":[91],"proposed":[93,129],"by":[95],"estimating":[96],"advantage":[98],"difference":[101],"terminal":[103],"success":[104],"rates":[105],"between":[106],"current-policy":[107],"continuations":[108],"sampled":[109],"before":[110],"after":[112],"segment.":[114,130],"Positive":[115],"estimates":[116],"are":[117],"then":[118],"incorporated":[119],"into":[120],"GRPO":[122,173],"within":[127],"By":[131],"using":[132],"judgment":[134],"only":[135],"select":[137],"where":[138],"verify,":[140],"grounds":[142],"local":[143],"observed":[146],"outcomes":[147],"without":[148,199],"exhaustively":[149],"evaluating":[150],"every":[151],"state.":[153],"Across":[154],"ALFWorld,":[155],"WebShop,":[156],"SearchQA,":[158],"achieves":[160],"strongest":[162],"average":[163],"performance":[164],"at":[165],"both":[166],"scales,":[168],"with":[169,193],"relative":[170],"improvements":[171],"over":[172],"9.91%":[175],"7.12%":[177],"Qwen3.5-2B":[179],"Qwen3.5-4B,":[181],"respectively.":[182],"Further":[183],"analyses":[184],"demonstrate":[185],"informed":[187],"selection":[189],"improves":[190],"modest":[194],"additional":[195],"generation":[196],"overhead,":[197],"even":[198],"frontier-scale":[201],"model,":[203],"highlighting":[204],"effectiveness":[206],"efficiency":[208],"selectively":[210],"targeting":[211]},"counts_by_year":[],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2026-10-01T00:00:00"}