{"id":"https://openalex.org/W7219155356","doi":null,"title":"Agent Evaluation Reliability: More Tasks Won't (Always) Fix An Agent Leaderboard","display_name":"Agent Evaluation Reliability: More Tasks Won't (Always) Fix An Agent Leaderboard","publication_year":2026,"publication_date":"2026-09-30","ids":{"openalex":"https://openalex.org/W7219155356"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.00651","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.00651","pdf_url":"https://arxiv.org/pdf/2610.00651","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.00651","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120302930","display_name":"Michael Hardy","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hardy, Michael","raw_affiliation_strings":["University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114427766","display_name":"Ruhana Azam","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Azam, Ruhana","raw_affiliation_strings":["Stanford University , * hardym [ ] stanford \u2022edu \u2663"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University , * hardym [ ] stanford \u2022edu \u2663","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059838123","display_name":"Anka Reuel","orcid":"https://orcid.org/0000-0002-7913-9296","observed_orcids":["https://orcid.org/0000-0002-7913-9296"]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Reuel, Anka","raw_affiliation_strings":["University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068326377","display_name":"Mykel J. Kochenderfer","orcid":"https://orcid.org/0000-0002-7238-9663","observed_orcids":["https://orcid.org/0000-0002-7238-9663"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kochenderfer, Mykel","raw_affiliation_strings":["Stanford University , * hardym [ ] stanford \u2022edu \u2663"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University , * hardym [ ] stanford \u2022edu \u2663","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5091266570","display_name":"Sanmi Koyejo","orcid":"https://orcid.org/0000-0002-4023-419X","observed_orcids":["https://orcid.org/0000-0002-4023-419X"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Koyejo, Sanmi","raw_affiliation_strings":["Stanford University , * hardym [ ] stanford \u2022edu \u2663"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University , * hardym [ ] stanford \u2022edu \u2663","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2224999964237213,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2224999964237213,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.08990000188350677,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.07530000060796738,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-agent-evaluation","display_name":"LLM agent evaluation","score":0.7310000061988831},{"id":"https://openalex.org/keywords/assessment-reliability","display_name":"assessment reliability","score":0.703000009059906},{"id":"https://openalex.org/keywords/model-ranking","display_name":"model ranking","score":0.367000013589859},{"id":"https://openalex.org/keywords/evaluation-design","display_name":"evaluation design","score":0.5569999814033508}],"concepts":[{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.835099995136261},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.711899995803833},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6909000277519226},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6766999959945679},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6284000277519226},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5702999830245972},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.5382999777793884},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.44690001010894775},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.4332999885082245},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3804999887943268},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.3386000096797943},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3357999920845032},{"id":"https://openalex.org/C8505890","wikidata":"https://www.wikidata.org/wiki/Q605095","display_name":"Budget constraint","level":2,"score":0.3253999948501587},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.2985999882221222},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2978000044822693},{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.2865000069141388},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.28360000252723694},{"id":"https://openalex.org/C37381756","wikidata":"https://www.wikidata.org/wiki/Q20203288","display_name":"Representativeness heuristic","level":2,"score":0.2815000116825104},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.28110000491142273},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.27059999108314514},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.2705000042915344},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.25429999828338623}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:arXiv.org:2610.00651","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.00651","pdf_url":"https://arxiv.org/pdf/2610.00651","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.00651","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.00651","pdf_url":"https://arxiv.org/pdf/2610.00651","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7219155356.pdf","grobid_xml":"https://content.openalex.org/works/W7219155356.grobid-xml"},"referenced_works_count":30,"referenced_works":["https://openalex.org/W974012311","https://openalex.org/W1964218847","https://openalex.org/W2037070511","https://openalex.org/W2145482311","https://openalex.org/W2170454983","https://openalex.org/W2203714058","https://openalex.org/W2996229661","https://openalex.org/W3022123134","https://openalex.org/W3214743865","https://openalex.org/W3216651781","https://openalex.org/W4238804190","https://openalex.org/W4293168481","https://openalex.org/W4385327683","https://openalex.org/W4394947078","https://openalex.org/W4399759481","https://openalex.org/W4399836817","https://openalex.org/W4400337927","https://openalex.org/W4403324055","https://openalex.org/W4403707291","https://openalex.org/W4415257516","https://openalex.org/W4417114362","https://openalex.org/W4417136556","https://openalex.org/W4417172194","https://openalex.org/W4417233452","https://openalex.org/W6929653740","https://openalex.org/W7128779957","https://openalex.org/W7130615426","https://openalex.org/W7162459404","https://openalex.org/W7166821844","https://openalex.org/W7170073213"],"related_works":["https://openalex.org/W7212121220","https://openalex.org/W7127582359","https://openalex.org/W7164744345","https://openalex.org/W7140685886","https://openalex.org/W7131893979","https://openalex.org/W7165363227","https://openalex.org/W7119224602","https://openalex.org/W7214170023","https://openalex.org/W7212052182","https://openalex.org/W4412877164"],"abstract_inverted_index":{"Agent":[0,87],"evaluations":[1,57],"are":[2,124],"increasingly":[3],"used":[4],"to":[5,81,99,212,226],"compare":[6],"LLMs":[7],"and":[8,60,78,89,219,248],"inform":[9],"deployment":[10],"decisions,":[11],"yet":[12],"ranks":[13,42],"can":[14,108,138,192,220],"reflect":[15],"not":[16,46],"only":[17],"the":[18,22,25,30,85,100,118,215,232,253],"model":[19,147],"but":[20],"also":[21],"effects":[23,152],"of":[24,173,255],"evaluation":[26,39,63,250],"conditions":[27],"such":[28],"as":[29],"scaffolds":[31,145],"or":[32,239],"tasks.":[33],"This":[34],"makes":[35],"reliability":[36,130,142,172,209],"claim-dependent:":[37],"an":[38],"that":[40,107,150,257],"reliably":[41,47,58,126],"deployed":[43],"systems":[44,123],"may":[45],"rank":[48],"underlying":[49],"models.":[50],"We":[51,67,112],"ask":[52],"which":[53],"conclusions":[54],"current":[55],"agent":[56,76],"support":[59],"what":[61,236,244],"additional":[62],"would":[64],"improve":[65,170,193],"them.":[66],"develop":[68],"a":[69,174,237],"Bayesian":[70],"variance-decomposition":[71],"framework":[72,93],"for":[73],"sparse,":[74],"imbalanced":[75],"leaderboards":[77],"apply":[79],"it":[80],"22":[82],"benchmarks":[83,191,206],"from":[84,103,210],"Holistic":[86],"Leaderboard":[88],"Harbor":[90],"Index.":[91],"The":[92],"separates":[94],"signal,":[95],"performance":[96],"differences":[97],"relevant":[98],"intended":[101,233],"claim,":[102],"noise,":[104],"irrelevant":[105],"variation":[106],"still":[109],"change":[110,139],"rankings.":[111],"find:":[113],"(1)":[114],"Reliability":[115],"depends":[116],"on":[117,252],"measurement":[119],"goal.":[120],"Fixed":[121],"model-scaffold":[122],"ranked":[125],"(0.935-0.994),":[127],"while":[128],"underlying-model":[129],"is":[131,182],"substantially":[132,154],"lower":[133,197],"(0.148-0.841).":[134],"(2)":[135],"Scaffold":[136],"choice":[137],"conclusions.":[140],"Inter-scaffold":[141],"measures":[143],"whether":[144],"preserve":[146],"rankings,":[148],"showing":[149],"scaffold":[151,186],"vary":[153],"across":[155,201],"evaluations.":[156],"(3)":[157],"More":[158],"tasks":[159,169],"cannot":[160],"resolve":[161],"all":[162],"uncertainty.":[163],"Even":[164],"infinitely":[165],"many":[166],"similarly":[167],"constructed":[168],"model-ranking":[171],"benchmark":[175],"by":[176,184,224],"at":[177,196,214],"most":[178],"0.097":[179],"when":[180],"uncertainty":[181,256],"dominated":[183],"limited":[185],"coverage.":[187],"(4)":[188],"Pooling":[189],"diverse":[190,202],"cross-task":[194],"rankings":[195,200],"cost.":[198],"For":[199],"agentic":[203],"tasks,":[204],"pooling":[205],"raises":[207],"projected":[208,222],"0.44":[211],"0.75":[213],"same":[216],"task":[217],"budget":[218,251],"reduce":[221],"cost":[223],"up":[225],"83\\%.":[227],"Evaluation":[228],"design":[229],"should":[230,241],"follow":[231],"claim:":[234],"identify":[235],"score":[238],"ranking":[240],"mean,":[242],"diagnose":[243],"limits":[245],"its":[246],"reliability,":[247],"spend":[249],"sources":[254],"matter.":[258]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-04T00:00:00"}