{"id":"https://openalex.org/W7166204187","doi":"https://doi.org/10.48550/arxiv.2606.26158","title":"Life After Benchmark Saturation: A Case Study of CORE-Bench","display_name":"Life After Benchmark Saturation: A Case Study of CORE-Bench","publication_year":2026,"publication_date":"2026-06-23","ids":{"openalex":"https://openalex.org/W7166204187","doi":"https://doi.org/10.48550/arxiv.2606.26158"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.26158","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.26158","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.26158","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139425190","display_name":"Nitya Nadgir","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nadgir, Nitya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063345981","display_name":"Sayash Kapoor","orcid":"https://orcid.org/0000-0001-5695-280X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kapoor, Sayash","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088475090","display_name":"K. Liu","orcid":"https://orcid.org/0000-0002-6491-5862"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Kangheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139463648","display_name":"Peter Kirgis","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kirgis, Peter","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139461352","display_name":"Matilda Orona","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Orona, Matilda","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139450298","display_name":"Stephan Rabanser","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rabanser, Stephan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007172052","display_name":"Tilman Bayer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bayer, Tilman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134752883","display_name":"Abhishek Shetty","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shetty, Abhishek","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139430567","display_name":"Yue Ling","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ling, Yue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134638805","display_name":"Derrick Chan-Sew","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chan-Sew, Derrick","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084061298","display_name":"Rumi Nakagawa","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nakagawa, Rumi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025772684","display_name":"Saiteja Utpala","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Utpala, Saiteja","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139455612","display_name":"Zachary S. Siegel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Siegel, Zachary S.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139446061","display_name":"Arvind Narayanan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Narayanan, Arvind","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.3889999985694885,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.3889999985694885,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.08489999920129776,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.05829999968409538,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.870199978351593},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.7656999826431274},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6844000220298767},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5493000149726868},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.4875999987125397},{"id":"https://openalex.org/keywords/limit","display_name":"Limit (mathematics)","score":0.4564000070095062},{"id":"https://openalex.org/keywords/construct-validity","display_name":"Construct validity","score":0.45010000467300415}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.870199978351593},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.7656999826431274},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6844000220298767},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6736999750137329},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5493000149726868},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4875999987125397},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.4564000070095062},{"id":"https://openalex.org/C49453240","wikidata":"https://www.wikidata.org/wiki/Q1592163","display_name":"Construct validity","level":3,"score":0.45010000467300415},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.445499986410141},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43209999799728394},{"id":"https://openalex.org/C174106493","wikidata":"https://www.wikidata.org/wiki/Q1057880","display_name":"External validity","level":2,"score":0.42669999599456787},{"id":"https://openalex.org/C3019813237","wikidata":"https://www.wikidata.org/wiki/Q65089264","display_name":"Model validation","level":2,"score":0.39329999685287476},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.3865000009536743},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3824999928474426},{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.357699990272522},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.33320000767707825},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.2759999930858612},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.2639000117778778},{"id":"https://openalex.org/C2780873155","wikidata":"https://www.wikidata.org/wiki/Q392811","display_name":"Agent-based model","level":2,"score":0.25270000100135803}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.26158","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.26158","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.26158","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.26158","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"When":[0],"a":[1,12,64,73,150,167,173,202],"benchmark's":[2],"accuracy":[3,22,92,132],"saturates,":[4],"it":[5],"is":[6],"often":[7],"retired":[8],"and":[9,23,55,120,144,193],"replaced":[10],"with":[11,109],"more":[13,203],"challenging":[14],"version.":[15],"We":[16,60,113,165],"show":[17],"that":[18,78,104,130],"this":[19],"approach":[20],"privileges":[21],"misses":[24],"the":[25,46,50,53,187,207],"opportunity":[26],"to":[27,76,98,107,154,181,206],"study":[28,75],"six":[29],"other":[30,196],"key":[31],"dimensions":[32,83],"of":[33,49,69,175,183],"agent":[34,88],"performance:":[35],"construct":[36,99],"validity":[37,100],"issues":[38],"such":[39],"as":[40,72],"shortcuts,":[41],"out-of-distribution":[42,122],"generalizability,":[43],"efficiency,":[44,140],"reliability,":[45,141],"relative":[47],"importance":[48],"model":[51,142],"versus":[52],"scaffold,":[54],"uplift":[56,156],"from":[57,157],"human-agent":[58,158],"collaboration.":[59],"use":[61],"CORE-Bench":[62,102,118,125,134],"Hard,":[63],"benchmark":[65],"for":[66,138],"computational":[67,162],"reproducibility":[68,163],"scientific":[70],"code,":[71],"case":[74],"demonstrate":[77],"measuring":[79,139],"agents":[80],"along":[81],"these":[82],"yields":[84],"meaningful":[85],"insights":[86],"into":[87],"performance":[89],"even":[90],"after":[91],"saturates.":[93],"First,":[94],"we":[95,128,148],"surface":[96],"threats":[97],"in":[101],"Hard":[103],"are":[105],"difficult":[106],"anticipate":[108],"less":[110],"capable":[111],"agents.":[112],"introduce":[114],"an":[115,121],"improved":[116],"benchmark,":[117],"v1.1,":[119],"task":[123],"suite,":[124],"OOD.":[126],"Second,":[127],"find":[129,166],"despite":[131],"saturation,":[133],"v1.1":[135],"remains":[136],"useful":[137],"performance,":[143],"scaffold":[145],"performance.":[146],"Finally,":[147],"conduct":[149],"small-scale":[151],"randomized":[152],"experiment":[153],"measure":[155],"collaboration":[159],"on":[160],"real-world":[161],"tasks.":[164],"statistically":[168],"significant":[169],"speedup":[170],"by":[171],"about":[172],"factor":[174],"two":[176],"--":[177,192],"likely":[178],"underestimated":[179],"due":[180],"one-fifth":[182],"human-only":[184],"reproductions":[185],"reaching":[186],"time":[188],"limit":[189],"before":[190],"completing":[191],"describe":[194],"various":[195],"findings.":[197],"Together,":[198],"our":[199],"contributions":[200],"present":[201],"rigorous":[204],"alternative":[205],"dominant":[208],"accuracy-centric":[209],"evaluation":[210],"paradigm.":[211]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-27T00:00:00"}