{"id":"https://openalex.org/W7160897323","doi":"https://doi.org/10.48550/arxiv.2605.10448","title":"Can Agent Benchmarks Support Their Scores? Evidence-Supported Bounds for Interactive-Agent Evaluation","display_name":"Can Agent Benchmarks Support Their Scores? Evidence-Supported Bounds for Interactive-Agent Evaluation","publication_year":2026,"publication_date":"2026-05-11","ids":{"openalex":"https://openalex.org/W7160897323","doi":"https://doi.org/10.48550/arxiv.2605.10448"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.10448","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.10448","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.10448","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135954169","display_name":"Shanshan Gao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Shanshan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135984049","display_name":"Liyi Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Liyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.2020999938249588,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.2020999938249588,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.1151999980211258,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.09960000216960907,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/outcome","display_name":"Outcome (game theory)","score":0.8568000197410583},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7347999811172485},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.669700026512146},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5662000179290771},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5315999984741211},{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.4912000000476837},{"id":"https://openalex.org/keywords/ask-price","display_name":"Ask price","score":0.444599986076355},{"id":"https://openalex.org/keywords/layer","display_name":"Layer (electronics)","score":0.41040000319480896},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.41019999980926514}],"concepts":[{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.8568000197410583},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7347999811172485},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7020999789237976},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.669700026512146},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5662000179290771},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5315999984741211},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.4912000000476837},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.444599986076355},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.41040000319480896},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.41019999980926514},{"id":"https://openalex.org/C2779356329","wikidata":"https://www.wikidata.org/wiki/Q922625","display_name":"Checklist","level":2,"score":0.3804999887943268},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3783000111579895},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.37770000100135803},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.3693999946117401},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.3617999851703644},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.3467999994754791},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32690000534057617},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3158999979496002},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.30790001153945923},{"id":"https://openalex.org/C66905080","wikidata":"https://www.wikidata.org/wiki/Q17005494","display_name":"Binary classification","level":3,"score":0.2773999869823456},{"id":"https://openalex.org/C149629883","wikidata":"https://www.wikidata.org/wiki/Q660926","display_name":"Fraction (chemistry)","level":2,"score":0.2680000066757202},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.260699987411499},{"id":"https://openalex.org/C112758219","wikidata":"https://www.wikidata.org/wiki/Q16038819","display_name":"Duration (music)","level":2,"score":0.2563999891281128},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2533999979496002},{"id":"https://openalex.org/C97399411","wikidata":"https://www.wikidata.org/wiki/Q825367","display_name":"Coin flipping","level":2,"score":0.2506999969482422}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.10448","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.10448","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.10448","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.10448","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Interactive":[0],"agent":[1,5,60,75],"benchmarks":[2],"map":[3],"an":[4,117],"run":[6,37,85,164],"to":[7,24,147,161],"a":[8,41,84,158,202],"binary":[9],"outcome":[10,12,54,109,118,151,216],"through":[11],"checks.":[13],"When":[14],"these":[15],"checks":[16],"rely":[17],"on":[18,100,105,219],"surface":[19],"level":[20],"signals":[21],"or":[22,130,176,197],"fail":[23],"capture":[25],"the":[26,36,53,59,68,74,79,90,106,149,207,215],"agent's":[27],"actual":[28],"action":[29],"path,":[30],"they":[31],"cannot":[32],"reliably":[33],"determine":[34],"whether":[35,46],"succeeded.":[38],"For":[39],"example,":[40],"benchmark":[42],"task":[43,101],"may":[44,76],"ask":[45],"Alice's":[47],"shipping":[48],"address":[49,112],"was":[50],"changed,":[51],"while":[52],"check":[55],"only":[56,99],"verifies":[57],"that":[58,67,185],"clicked":[61],"\"Save.\"":[62],"This":[63],"does":[64],"not":[65,98],"guarantee":[66],"intended":[69],"state":[70],"change":[71],"occurred,":[72],"since":[73],"have":[77],"modified":[78],"wrong":[80],"record.":[81],"Treating":[82],"such":[83],"as":[86],"successful":[87],"therefore":[88],"makes":[89],"reported":[91],"score":[92,183],"misleading.":[93],"Benchmark":[94],"quality":[95],"thus":[96],"depends":[97],"design,":[102],"but":[103],"also":[104],"reliability":[107],"of":[108,168],"detection.":[110],"We":[111,213],"this":[113],"problem":[114],"by":[115],"introducing":[116],"evidence":[119,170,181,217],"reporting":[120],"layer":[121,133,218],"for":[122,152],"existing":[123],"benchmarks,":[124],"without":[125],"modifying":[126],"their":[127],"tasks,":[128],"agents,":[129],"evaluators.":[131],"The":[132,231],"performs":[134],"three":[135,169],"functions.":[136],"First,":[137],"before":[138],"scoring,":[139],"it":[140,156,179],"specifies":[141],"which":[142],"stored":[143],"artifacts":[144],"are":[145],"required":[146],"verify":[148],"claimed":[150],"each":[153,162],"case.":[154],"Second,":[155],"applies":[157],"locked":[159],"checklist":[160],"completed":[163],"and":[165,229],"assigns":[166],"one":[167],"labels:":[171],"Evidence":[172,174],"Pass,":[173],"Fail,":[175],"Unknown.":[177],"Third,":[178],"reports":[180,233],"supported":[182],"bounds":[184],"quantify":[186],"uncertainty":[187],"arising":[188],"from":[189],"Unknown":[190],"cases.":[191],"Rather":[192],"than":[193],"silently":[194],"counting,":[195],"discarding,":[196],"hiding":[198],"uncertain":[199],"cases":[200],"inside":[201],"single":[203],"aggregate":[204],"success":[205],"rate,":[206],"framework":[208],"keeps":[209],"them":[210],"explicitly":[211],"visible.":[212],"evaluate":[214],"five":[220],"public":[221],"benchmarks:":[222],"ANDROIDWORLD,":[223],"AGENTDOJO,":[224],"APPWORLD,":[225],"tau3":[226],"bench":[227],"retail,":[228],"MINIWOB.":[230],"resulting":[232],"separate":[234],"several":[235],"empirically":[236],"distinct":[237],"failure":[238],"modes.":[239]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-13T00:00:00"}