{"id":"https://openalex.org/W7165936402","doi":"https://doi.org/10.48550/arxiv.2606.25556","title":"BiPACE: Bisimulation-Guided Policy Optimization with Action Counterfactual Estimation for LLM Agents","display_name":"BiPACE: Bisimulation-Guided Policy Optimization with Action Counterfactual Estimation for LLM Agents","publication_year":2026,"publication_date":"2026-06-24","ids":{"openalex":"https://openalex.org/W7165936402","doi":"https://doi.org/10.48550/arxiv.2606.25556"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.25556","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.25556","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.25556","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139352255","display_name":"Hanyang Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Hanyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139321625","display_name":"Weijieying Ren","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ren, Weijieying","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139322315","display_name":"Yuxiang Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yuxiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139380104","display_name":"Ding Cao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Ding","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139361616","display_name":"Zhizhao Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Zhizhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139372068","display_name":"Ke Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Ke","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5053042660","display_name":"Tianxiang Zhao","orcid":"https://orcid.org/0000-0003-4504-7809"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Tianxiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.6381999850273132,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.6381999850273132,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.0812000036239624,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.023499999195337296,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.745199978351593},{"id":"https://openalex.org/keywords/counterfactual-thinking","display_name":"Counterfactual thinking","score":0.6674000024795532},{"id":"https://openalex.org/keywords/proxy","display_name":"Proxy (statistics)","score":0.5144000053405762},{"id":"https://openalex.org/keywords/observational-equivalence","display_name":"Observational equivalence","score":0.5080999732017517},{"id":"https://openalex.org/keywords/singleton","display_name":"Singleton","score":0.47859999537467957},{"id":"https://openalex.org/keywords/equivalence","display_name":"Equivalence (formal languages)","score":0.4564000070095062},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.3855000138282776},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.3343000113964081}],"concepts":[{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.745199978351593},{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.6674000024795532},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.5144000053405762},{"id":"https://openalex.org/C2777721189","wikidata":"https://www.wikidata.org/wiki/Q17121921","display_name":"Observational equivalence","level":2,"score":0.5080999732017517},{"id":"https://openalex.org/C117354338","wikidata":"https://www.wikidata.org/wiki/Q1165112","display_name":"Singleton","level":3,"score":0.47859999537467957},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.476500004529953},{"id":"https://openalex.org/C2780069185","wikidata":"https://www.wikidata.org/wiki/Q7977945","display_name":"Equivalence (formal languages)","level":2,"score":0.4564000070095062},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.4244000017642975},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.40369999408721924},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.3855000138282776},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3578999936580658},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.3343000113964081},{"id":"https://openalex.org/C42812","wikidata":"https://www.wikidata.org/wiki/Q1082910","display_name":"Partition (number theory)","level":2,"score":0.3174999952316284},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.30630001425743103},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2896000146865845},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2863999903202057},{"id":"https://openalex.org/C164172150","wikidata":"https://www.wikidata.org/wiki/Q1782585","display_name":"Consistent estimator","level":4,"score":0.28529998660087585},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.2806999981403351},{"id":"https://openalex.org/C6177178","wikidata":"https://www.wikidata.org/wiki/Q10998070","display_name":"Discounting","level":2,"score":0.26919999718666077},{"id":"https://openalex.org/C2781181686","wikidata":"https://www.wikidata.org/wiki/Q4226068","display_name":"Coherence (philosophical gambling strategy)","level":2,"score":0.2671000063419342},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.2599000036716461},{"id":"https://openalex.org/C166109690","wikidata":"https://www.wikidata.org/wiki/Q4677422","display_name":"Action selection","level":3,"score":0.2587999999523163},{"id":"https://openalex.org/C41426520","wikidata":"https://www.wikidata.org/wiki/Q1192065","display_name":"Point estimation","level":2,"score":0.2583000063896179},{"id":"https://openalex.org/C119043178","wikidata":"https://www.wikidata.org/wiki/Q320723","display_name":"Covariate","level":2,"score":0.25760000944137573},{"id":"https://openalex.org/C162838799","wikidata":"https://www.wikidata.org/wiki/Q596077","display_name":"Counterexample","level":2,"score":0.257099986076355},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.25519999861717224},{"id":"https://openalex.org/C138777275","wikidata":"https://www.wikidata.org/wiki/Q6884054","display_name":"Mixing (physics)","level":2,"score":0.2513999938964844}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.25556","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.25556","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.25556","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.25556","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.6388768553733826}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Stepwise":[0],"group-based":[1],"RL":[2],"is":[3,27,64,83,229,256],"an":[4,135],"attractive":[5],"way":[6],"to":[7,21,181,247],"train":[8],"long-horizon":[9],"LLM":[10],"agents":[11],"without":[12,114],"a":[13,57,79,106,116,167,232],"learned":[14],"critic:":[15],"it":[16,40,204,215,238],"reuses":[17],"multiple":[18],"sampled":[19],"rollouts":[20],"estimate":[22],"local":[23,168],"advantages.":[24],"Its":[25],"weakness":[26],"less":[28],"visible":[29],"but":[30],"more":[31],"fundamental:":[32],"every":[33,192],"group-relative":[34],"estimator":[35,109],"assumes":[36],"that":[37,49,110,141],"the":[38,68,87,130,144,188,199,240],"steps":[39,125],"compares":[41],"are":[42],"equivalent":[43],"for":[44,139],"credit":[45,59],"assignment.":[46],"We":[47,96],"show":[48],"current":[50],"agentic":[51],"variants":[52],"violate":[53],"this":[54],"assumption":[55],"through":[56],"state-action":[58],"mismatch.":[60],"The":[61,225,254],"observation-hash":[62],"partition":[63],"overly":[65],"fine":[66],"on":[67,86,191,211],"state":[69],"side,":[70,89],"creating":[71],"singleton":[72,145],"groups":[73],"with":[74,93,102],"zero":[75],"step-level":[76],"signal,":[77],"while":[78],"single":[80,233],"within-group":[81],"mean":[82],"too":[84],"coarse":[85],"action":[88],"mixing":[90],"state-value":[91],"estimation":[92],"action-specific":[94],"credit.":[95],"introduce":[97],"BiPACE":[98],"(Bisimulation-Guided":[99],"Policy":[100],"Optimization":[101],"Action":[103],"Counterfactual":[104],"Estimation),":[105],"drop-in":[107],"advantage":[108],"fixes":[111],"both":[112,222],"sides":[113],"adding":[115],"critic,":[117],"auxiliary":[118],"loss,":[119],"or":[120],"extra":[121],"rollouts.":[122],"BiGPO":[123],"clusters":[124],"by":[126,148],"cosine":[127],"distance":[128],"in":[129],"actor's":[131],"own":[132],"hidden-state":[133],"geometry,":[134],"empirical":[136],"policy-induced":[137],"proxy":[138],"bisimulation":[140],"substantially":[142],"lowers":[143],"rate":[146],"left":[147],"observation":[149],"hashing.":[150],"PACE":[151],"then":[152],"recenters":[153],"returns":[154],"within":[155,198],"each":[156],"behavioral":[157,249],"cluster":[158],"using":[159],"action-conditioned":[160],"peer":[161],"baselines;":[162],"its":[163],"Q-style":[164],"instance":[165],"estimates":[166],"Q(s,a)-V(s)":[169],"nonparametrically.":[170],"On":[171,202],"ALFWorld/Qwen2.5-7B,":[172],"BiPACE_Q":[173],"raises":[174],"overall":[175],"validation":[176],"success":[177],"from":[178,244],"GiGPO's":[179,208],"90.8":[180],"$97.1\\pm0.9$":[182],"over":[183,217],"three":[184],"seeds,":[185],"and":[186,210,213,219],"crosses":[187],"95%":[189],"threshold":[190],"seed,":[193],"which":[194],"GiGPO":[195,220],"never":[196],"does":[197],"same":[200],"budget.":[201],"Qwen2.5-1.5B":[203],"reaches":[205],"$93.5\\pm1.2$":[206],"versus":[207],"86.7,":[209],"WebShop":[212],"TextCraft":[214],"improves":[216],"GRPO":[218],"at":[221,258],"model":[223],"scales.":[224],"measured":[226],"BiPACE-specific":[227],"overhead":[228],"11.3%":[230],"of":[231],"training-step":[234],"wall":[235],"time.":[236],"Yet":[237],"changes":[239],"estimator's":[241],"comparison":[242],"unit":[243],"surface":[245],"identity":[246],"approximate":[248],"equivalence":[250],"plus":[251],"action-side":[252],"counterfactuals.":[253],"code":[255],"available":[257],"https://github.com/TianxiangZhao/BiPACE.":[259]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-26T00:00:00"}