{"id":"https://openalex.org/W7214705437","doi":"https://doi.org/10.48550/arxiv.2609.35215","title":"ASCT: Attentive Search over Counterfactual Trees for Credit Assignment in Agentic Reinforcement Learning","display_name":"ASCT: Attentive Search over Counterfactual Trees for Credit Assignment in Agentic Reinforcement Learning","publication_year":2026,"publication_date":"2026-09-28","ids":{"openalex":"https://openalex.org/W7214705437","doi":"https://doi.org/10.48550/arxiv.2609.35215"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.35215","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35215","pdf_url":"https://arxiv.org/pdf/2609.35215","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.35215","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5153432674","display_name":"Yang Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Li, Yang","raw_affiliation_strings":["University of Hong Kong (hku.hk"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Hong Kong (hku.hk","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153499773","display_name":"Jinhan Yang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yang, Jinhan","raw_affiliation_strings":["The Chinese University of Hong Kong (cuhk"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong (cuhk","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153404299","display_name":"hai liu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I41317344","display_name":"Jiangxi Science and Technology Normal University","ror":"https://ror.org/04r1zkp10","country_code":"CN","type":"education","lineage":["https://openalex.org/I41317344"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"liu, hai","raw_affiliation_strings":["Jiangxi Science and Technology Normal University (jxstnu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jiangxi Science and Technology Normal University (jxstnu","institution_ids":["https://openalex.org/I41317344"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153538245","display_name":"Di Wan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wan, Di","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153440248","display_name":"Xiyu Chen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Xiyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5150915323","display_name":"Zongsi Xu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xu, Zongsi","raw_affiliation_strings":["University of Hong Kong (hku.hk"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Hong Kong (hku.hk","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126461122","display_name":"Tuo Zhou","orcid":"https://orcid.org/0000-0003-3913-042X","observed_orcids":["https://orcid.org/0000-0003-3913-042X"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhou, Tuo","raw_affiliation_strings":["University of Hong Kong (hku.hk"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Hong Kong (hku.hk","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153549152","display_name":"Sheng Zhong","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhong, Sheng","raw_affiliation_strings":["University of Hong Kong (hku.hk"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Hong Kong (hku.hk","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153520462","display_name":"Sergey Volkov","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Volkov, Sergey","raw_affiliation_strings":["University of Hong Kong (hku.hk"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Hong Kong (hku.hk","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153493610","display_name":"Ye Luo","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Luo, Ye","raw_affiliation_strings":["University of Hong Kong (hku.hk"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Hong Kong (hku.hk","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5153435015","display_name":"Hao Sun","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I180726961","display_name":"Shenzhen University","ror":"https://ror.org/01vy4gh70","country_code":"CN","type":"education","lineage":["https://openalex.org/I180726961"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sun, Hao","raw_affiliation_strings":["Shenzhen University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shenzhen University","institution_ids":["https://openalex.org/I180726961"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9771000146865845,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9771000146865845,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.004699999932199717,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.0034000000450760126,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/credit-assignment","display_name":"credit assignment","score":0.7839999794960022},{"id":"https://openalex.org/keywords/agentic-reinforcement-learning","display_name":"agentic reinforcement learning","score":0.9079999923706055},{"id":"https://openalex.org/keywords/uct","display_name":"UCT","score":0.6190000176429749},{"id":"https://openalex.org/keywords/retrieval-augmented-generation","display_name":"retrieval-augmented generation","score":0.8190000057220459},{"id":"https://openalex.org/keywords/hotpotqa","display_name":"HotpotQA","score":0.9020000100135803},{"id":"https://openalex.org/keywords/ppo","display_name":"PPO","score":0.5929999947547913},{"id":"https://openalex.org/keywords/policy-transfer","display_name":"policy transfer","score":0.35600000619888306},{"id":"https://openalex.org/keywords/execution-cost","display_name":"execution cost","score":0.3479999899864197}],"concepts":[{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.9452999830245972},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6319000124931335},{"id":"https://openalex.org/C71889745","wikidata":"https://www.wikidata.org/wiki/Q1783264","display_name":"Counterfactual conditional","level":3,"score":0.5382000207901001},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5224999785423279},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5120999813079834},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5116000175476074},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.4837000072002411},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.46399998664855957},{"id":"https://openalex.org/C45235069","wikidata":"https://www.wikidata.org/wiki/Q278425","display_name":"Table (database)","level":2,"score":0.46050000190734863},{"id":"https://openalex.org/C84525736","wikidata":"https://www.wikidata.org/wiki/Q831366","display_name":"Decision tree","level":2,"score":0.4537999927997589},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.30880001187324524},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.2985000014305115},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.28679999709129333},{"id":"https://openalex.org/C155108698","wikidata":"https://www.wikidata.org/wiki/Q1231081","display_name":"Randomized experiment","level":2,"score":0.271699994802475},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2696000039577484},{"id":"https://openalex.org/C2776175482","wikidata":"https://www.wikidata.org/wiki/Q1195816","display_name":"Transfer (computing)","level":2,"score":0.2639000117778778},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.2612000107765198},{"id":"https://openalex.org/C207024777","wikidata":"https://www.wikidata.org/wiki/Q621673","display_name":"Search tree","level":3,"score":0.25699999928474426},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.2556999921798706}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.35215","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35215","pdf_url":"https://arxiv.org/pdf/2609.35215","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.35215","doi":"https://doi.org/10.48550/arxiv.2609.35215","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.35215","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.35215","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35215","pdf_url":"https://arxiv.org/pdf/2609.35215","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214705437.pdf","grobid_xml":"https://content.openalex.org/works/W7214705437.grobid-xml"},"referenced_works_count":16,"referenced_works":["https://openalex.org/W1625390266","https://openalex.org/W2168405694","https://openalex.org/W2736601468","https://openalex.org/W2962938168","https://openalex.org/W3138773240","https://openalex.org/W3168867926","https://openalex.org/W4252076394","https://openalex.org/W4310923309","https://openalex.org/W4389422760","https://openalex.org/W4400337884","https://openalex.org/W4417252869","https://openalex.org/W7083493645","https://openalex.org/W7123563915","https://openalex.org/W7133510219","https://openalex.org/W7169608443","https://openalex.org/W7171498516"],"related_works":["https://openalex.org/W7223413470","https://openalex.org/W7166687498","https://openalex.org/W7166852645","https://openalex.org/W7154202256","https://openalex.org/W7168511125","https://openalex.org/W7203909526","https://openalex.org/W7214958905","https://openalex.org/W7160931659","https://openalex.org/W7214114218","https://openalex.org/W7204211622"],"abstract_inverted_index":{"Terminal":[0],"utility":[1,99,112],"evaluates":[2,41],"a":[3,24],"complete":[4],"agentic":[5,91],"workflow,":[6],"but":[7],"learning":[8,75],"requires":[9],"credit":[10,62],"for":[11,63,115],"the":[12,46,56,78,87,138,142],"decisions":[13],"within":[14],"it.":[15],"We":[16],"introduce":[17],"Attentive":[18],"Search":[19],"over":[20,100],"Counterfactual":[21],"Trees":[22],"(ASCT),":[23],"framework":[25],"that":[26],"turns":[27],"training-time":[28],"multi-step":[29],"search":[30],"into":[31],"local":[32],"action":[33],"credit.":[34],"At":[35],"actor-visited":[36],"states,":[37],"an":[38],"auxiliary":[39,130],"tree":[40],"alternative":[42],"legal":[43],"actions":[44],"from":[45],"same":[47],"recoverable":[48],"prefix.":[49],"Its":[50],"action-value":[51],"table":[52],"is":[53],"centered":[54],"by":[55],"frozen":[57],"actor's":[58],"probabilities":[59],"and":[60,83,103,122,125,134],"supplies":[61],"PPO":[64,102],"on":[65],"actor-sampled":[66],"trajectories.":[67],"This":[68],"protocol":[69],"connects":[70],"counterfactual":[71],"evaluation":[72],"to":[73],"policy":[74],"while":[76],"deploying":[77],"actor":[79],"alone.":[80],"Uniform,":[81],"UCT,":[82],"cost-aware":[84],"AgentUCT":[85],"instantiate":[86],"framework.":[88],"On":[89],"HotpotQA":[90],"retrieval-augmented":[92],"generation,":[93],"all":[94],"three":[95,107],"improve":[96],"mean":[97],"held-out":[98],"trajectory-return":[101],"workflow-adapted":[104],"VinePPO.":[105],"Across":[106],"seeds,":[108],"ASCT-AgentUCT":[109],"reaches":[110],"0.6187":[111],"versus":[113],"0.5939":[114],"VinePPO,":[116],"with":[117],"gains":[118],"in":[119],"answer":[120],"F1":[121],"execution":[123],"cost,":[124],"uses":[126],"50.3%":[127],"fewer":[128],"recorded":[129],"Qwen":[131],"tokens.":[132],"Transfer":[133],"component-description":[135],"studies":[136],"examine":[137],"learned":[139],"policies":[140],"beyond":[141],"training":[143],"setting.":[144]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-30T00:00:00"}