{"id":"https://openalex.org/W7225615684","doi":"https://doi.org/10.48550/arxiv.2611.09792","title":"Knowing When to Intervene: Causal Evaluation of AI Agents","display_name":"Knowing When to Intervene: Causal Evaluation of AI Agents","publication_year":2026,"publication_date":"2026-11-06","ids":{"openalex":"https://openalex.org/W7225615684","doi":"https://doi.org/10.48550/arxiv.2611.09792"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2611.09792","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.09792","pdf_url":"https://arxiv.org/pdf/2611.09792","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2611.09792","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067698930","display_name":"Ayush Sawarni","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sawarni, Ayush","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047300631","display_name":"Wenqian Xing","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xing, Wenqian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032482329","display_name":"Tijana Zrnic","orcid":"https://orcid.org/0000-0002-5112-8632","observed_orcids":["https://orcid.org/0000-0002-5112-8632"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zrnic, Tijana","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042177635","display_name":"Ramesh Johari","orcid":"https://orcid.org/0000-0002-3960-0770","observed_orcids":["https://orcid.org/0000-0002-3960-0770"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Johari, Ramesh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5017741738","display_name":"Vasilis Syrgkanis","orcid":"https://orcid.org/0000-0001-5047-8772","observed_orcids":["https://orcid.org/0000-0001-5047-8772"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Syrgkanis, Vasilis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.3508000075817108,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.3508000075817108,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.2874000072479248,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.12520000338554382,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/agentic-ai","display_name":"agentic AI","score":0.7450000047683716},{"id":"https://openalex.org/keywords/causal-effect-estimation","display_name":"causal effect estimation","score":0.5320000052452087},{"id":"https://openalex.org/keywords/randomized-intervention","display_name":"randomized intervention","score":0.8629999756813049},{"id":"https://openalex.org/keywords/agent-trajectories","display_name":"agent trajectories","score":0.6620000004768372},{"id":"https://openalex.org/keywords/ai-agent-evaluation","display_name":"AI agent evaluation","score":0.6179999709129333},{"id":"https://openalex.org/keywords/ai-safety","display_name":"AI safety","score":0.39100000262260437}],"concepts":[{"id":"https://openalex.org/C158600405","wikidata":"https://www.wikidata.org/wiki/Q5054566","display_name":"Causal inference","level":2,"score":0.7652999758720398},{"id":"https://openalex.org/C27415008","wikidata":"https://www.wikidata.org/wiki/Q7256382","display_name":"Psychological intervention","level":2,"score":0.6097999811172485},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5659999847412109},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5299999713897705},{"id":"https://openalex.org/C2780665704","wikidata":"https://www.wikidata.org/wiki/Q959298","display_name":"Intervention (counseling)","level":2,"score":0.5099999904632568},{"id":"https://openalex.org/C11671645","wikidata":"https://www.wikidata.org/wiki/Q5054567","display_name":"Causal model","level":2,"score":0.4772999882698059},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.47350001335144043},{"id":"https://openalex.org/C155108698","wikidata":"https://www.wikidata.org/wiki/Q1231081","display_name":"Randomized experiment","level":2,"score":0.4431000053882599},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.42829999327659607},{"id":"https://openalex.org/C115086926","wikidata":"https://www.wikidata.org/wiki/Q17004651","display_name":"Causal reasoning","level":3,"score":0.40560001134872437},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3637000024318695},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.3637000024318695},{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.3619000017642975},{"id":"https://openalex.org/C64543145","wikidata":"https://www.wikidata.org/wiki/Q162942","display_name":"Intersection (aeronautics)","level":2,"score":0.35499998927116394},{"id":"https://openalex.org/C64357122","wikidata":"https://www.wikidata.org/wiki/Q1149766","display_name":"Causality (physics)","level":2,"score":0.34860000014305115},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.3474999964237213},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3456000089645386},{"id":"https://openalex.org/C71889745","wikidata":"https://www.wikidata.org/wiki/Q1783264","display_name":"Counterfactual conditional","level":3,"score":0.31220000982284546},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.30219998955726624},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.29600000381469727},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.2842000126838684},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2551000118255615},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.2515000104904175}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2611.09792","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.09792","pdf_url":"https://arxiv.org/pdf/2611.09792","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2611.09792","doi":"https://doi.org/10.48550/arxiv.2611.09792","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.09792","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2611.09792","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.09792","pdf_url":"https://arxiv.org/pdf/2611.09792","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7167608595","https://openalex.org/W3194235655","https://openalex.org/W7214938914","https://openalex.org/W4414434834","https://openalex.org/W7163588632","https://openalex.org/W4399695842","https://openalex.org/W7213626338","https://openalex.org/W7212985188","https://openalex.org/W7160917275","https://openalex.org/W7172200799"],"abstract_inverted_index":{"Current":[0],"evaluations":[1],"of":[2,14,22,72,86,108,113,142,154,173,177,199,245,262],"agentic":[3,68,201,249],"AI":[4,263],"systems":[5],"primarily":[6],"rely":[7],"on":[8,117],"benchmarks.":[9],"However,":[10],"the":[11,20,48,83,93,105,109,111,118,126,132,139,169,197,243,257,266],"static":[12],"nature":[13,107],"benchmarks":[15],"makes":[16],"them":[17],"uninformative":[18],"about":[19],"value":[21,71],"auditing":[23],"interventions":[24,74,144,160],"at":[25,242],"deployment":[26,261],"time.":[27],"For":[28],"reliable":[29,258],"deployment,":[30],"we":[31,166,185,221,252],"often":[32],"need":[33],"to":[34,43,104,191,215],"determine":[35],"within":[36,145],"an":[37,88,114],"agent's":[38,49,127],"trajectory":[39],"whether":[40,82],"or":[41,52,62,66,164],"not":[42],"intervene,":[44],"e.g.,":[45],"by":[46,207],"verifying":[47],"current":[50,119],"output":[51],"steering":[53],"it":[54],"toward":[55],"different":[56],"behavior,":[57],"using":[58],"either":[59],"human":[60],"review":[61],"more":[63],"costly":[64],"models":[65],"time-consuming":[67],"supervision.":[69],"The":[70,181],"such":[73,87,161],"is":[75,90,100,254],"fundamentally":[76],"a":[77,150,174,238],"causal":[78,84,98,140,171,246],"inference":[79,247],"question,":[80],"asking":[81],"effect":[85,99,172],"intervention":[89,115,179,224],"beneficial":[91],"for":[92,137,209,256],"final":[94],"outcome.":[95],"Estimating":[96],"this":[97],"particularly":[101],"challenging":[102],"due":[103],"dynamic":[106,170],"problem:":[110],"impact":[112],"depends":[116],"context":[120],"and":[121,219,231,248,259],"environment,":[122],"as":[123,125,162],"well":[124],"future":[128],"behavior.":[129],"We":[130],"develop":[131],"first":[133],"methodologically":[134],"grounded":[135],"framework":[136],"evaluating":[138],"effects":[141,225],"randomized":[143,159],"agent":[146,155,229],"trajectories.":[147],"By":[148],"leveraging":[149],"relatively":[151],"small":[152],"dataset":[153],"trajectories":[156],"collected":[157],"under":[158],"verification":[163],"screening,":[165],"can":[167,187,204],"estimate":[168,186],"rich":[175,239],"class":[176],"(contextual)":[178],"policies.":[180],"structural":[182],"parameters":[183],"that":[184,195,223],"also":[188],"be":[189,205],"used":[190,206],"construct":[192],"safety":[193],"leaderboards":[194],"capture":[196],"risk":[198],"various":[200],"actions,":[202],"which":[203,251],"practitioners":[208],"audit":[210],"prioritization.":[211],"Applying":[212],"our":[213,235],"methods":[214],"tau-bench":[216],"Retail,":[217],"ToolSandbox,":[218],"ImpossibleBench,":[220],"find":[222],"vary":[226],"substantially":[227],"across":[228],"actions":[230],"states.":[232],"More":[233],"broadly,":[234],"work":[236],"suggests":[237],"research":[240],"agenda":[241],"intersection":[244],"evaluation,":[250],"believe":[253],"essential":[255],"safe":[260],"agents":[264],"in":[265],"real":[267],"world.":[268]},"counts_by_year":[],"updated_date":"2026-11-11T12:16:26.678958","created_date":"2026-11-10T00:00:00"}