{"id":"https://openalex.org/W7167280081","doi":"https://doi.org/10.48550/arxiv.2607.02032","title":"PACE: A Proxy for Agentic Capability Evaluation","display_name":"PACE: A Proxy for Agentic Capability Evaluation","publication_year":2026,"publication_date":"2026-07-02","ids":{"openalex":"https://openalex.org/W7167280081","doi":"https://doi.org/10.48550/arxiv.2607.02032"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.02032","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.02032","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.02032","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134947889","display_name":"Y Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Yueqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139967659","display_name":"Lintang Sutawika","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sutawika, Lintang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139966716","display_name":"Jiarui Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jiarui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092050382","display_name":"Lindia Tjuatja","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tjuatja, Lindia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139955064","display_name":"Jiayi Geng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Geng, Jiayi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133363812","display_name":"Y H Xiao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao, Yunze","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033836068","display_name":"D Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139987217","display_name":"Aditya Bharat Soni","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Soni, Aditya Bharat","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140021682","display_name":"Vincent Lo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lo, Vincent","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139989800","display_name":"Xiang Yue","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yue, Xiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139993380","display_name":"Graham Neubig","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Neubig, Graham","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.19130000472068787,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.19130000472068787,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.10300000011920929,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.08009999990463257,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.8027999997138977},{"id":"https://openalex.org/keywords/proxy","display_name":"Proxy (statistics)","score":0.7803000211715698},{"id":"https://openalex.org/keywords/pace","display_name":"Pace","score":0.7748000025749207},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6509000062942505},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.45489999651908875},{"id":"https://openalex.org/keywords/regression","display_name":"Regression","score":0.37380000948905945}],"concepts":[{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.8027999997138977},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.7803000211715698},{"id":"https://openalex.org/C2777526511","wikidata":"https://www.wikidata.org/wiki/Q691543","display_name":"Pace","level":2,"score":0.7748000025749207},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6509000062942505},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5841000080108643},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.49129998683929443},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.45489999651908875},{"id":"https://openalex.org/C83546350","wikidata":"https://www.wikidata.org/wiki/Q1139051","display_name":"Regression","level":2,"score":0.37380000948905945},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.34459999203681946},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3427000045776367},{"id":"https://openalex.org/C2778565505","wikidata":"https://www.wikidata.org/wiki/Q2207566","display_name":"Spec#","level":2,"score":0.33340001106262207},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3287000060081482},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.311599999666214},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.30329999327659607},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.29109999537467957},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.2847000062465668}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.02032","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.02032","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.02032","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.02032","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.6227207779884338,"id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Evaluating":[0],"LLM":[1,33],"agents":[2],"on":[3,56,67,101,122,132],"benchmarks":[4,34,59,85,164,191],"like":[5],"SWE-Bench":[6],"and":[7,13,25,45,151,188,212,259],"GAIA":[8],"can":[9,20,60],"be":[10,61],"expensive,":[11],"time-consuming,":[12],"requires":[14],"complex":[15],"infrastructure.":[16],"A":[17],"single":[18],"evaluation":[19,75,228],"cost":[21],"thousands":[22],"of":[23,73,107,126,224,252,264],"dollars":[24],"take":[26],"days":[27],"to":[28,47,129,159,248],"complete.":[29],"In":[30,49],"contrast,":[31],"non-agentic":[32,91,190],"that":[35,82,117,175,193],"test":[36],"individual":[37],"capabilities":[38],"(e.g.,":[39],"reasoning,":[40],"code":[41],"generation)":[42],"are":[43],"fast":[44],"cheap":[46],"run.":[48],"this":[50,166],"paper,":[51,167],"we":[52,176],"investigate":[53],"whether":[54],"performance":[55,66,254],"expensive":[57],"agentic":[58,102,135,163,186,196,227,241,253],"accurately":[62],"predicted":[63],"by":[64,86,142],"the":[65,133,160,171,179,225,233,262],"a":[68,80,105,115,119,123],"small,":[69],"carefully":[70],"selected":[71,234],"subset":[72,125,138],"atomic":[74,111],"instances.":[76],"We":[77,156,230],"introduce":[78],"PACE,":[79],"framework":[81],"constructs":[83],"proxy":[84,173,235],"selecting":[87],"instances":[88,109,128],"from":[89],"existing":[90],"evaluations":[92],"whose":[93],"aggregate":[94],"scores":[95,121,197],"most":[96],"reliably":[97],"predict":[98],"model":[99,256],"performances":[100],"benchmarks.":[103],"Given":[104],"pool":[106],"candidate":[108],"spanning":[110],"capabilities,":[112],"PACE":[113,158,245],"fits":[114],"regression":[116],"maps":[118],"model's":[120],"compact":[124],"source":[127],"its":[130],"score":[131],"target":[134,162],"benchmark.":[136],"The":[137],"itself":[139],"is":[140],"curated":[141],"combining":[143],"two":[144],"complementary":[145],"instance-selection":[146],"strategies,":[147],"target-relevance":[148],"local":[149],"selection":[150],"globally":[152],"informative":[153],"global":[154],"selection.":[155],"apply":[157],"4":[161,185],"in":[165,178],"which":[168,238],"yields":[169],"PACE-Bench,":[170],"concrete":[172],"benchmark":[174,242],"evaluate":[177],"paper.":[180],"Experiments":[181],"across":[182],"14":[183],"models,":[184],"benchmarks,":[187],"19":[189],"show":[192],"PACE-Bench":[194],"predicts":[195],"with":[198],"leave-one-out":[199],"cross-validation":[200],"(LOOCV)":[201],"mean":[202],"absolute":[203],"error":[204],"(MAE)":[205],"under":[206],"4%,":[207],"Spearman":[208],"correlation":[209],"above":[210],"0.80,":[211],"pairwise":[213],"model-ranking":[214],"accuracy":[215],"around":[216],"85%,":[217],"all":[218],"at":[219],"much":[220],"less":[221],"than":[222],"1%":[223],"full":[226,265],"cost.":[229],"further":[231],"analyze":[232],"instances,":[236],"revealing":[237],"skills":[239],"each":[240],"uniquely":[243],"demands.":[244],"enables":[246],"practitioners":[247],"obtain":[249],"reliable":[250],"estimates":[251],"during":[255],"development,":[257],"selection,":[258],"routing,":[260],"without":[261],"overhead":[263],"agent":[266],"evaluation.":[267]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-04T00:00:00"}