{"id":"https://openalex.org/W7168361355","doi":"https://doi.org/10.48550/arxiv.2607.12227","title":"Rethinking the Evaluation of Harness Evolution for Agents","display_name":"Rethinking the Evaluation of Harness Evolution for Agents","publication_year":2026,"publication_date":"2026-07-14","ids":{"openalex":"https://openalex.org/W7168361355","doi":"https://doi.org/10.48550/arxiv.2607.12227"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.12227","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.12227","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.12227","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140793247","display_name":"Yike Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yike","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140869222","display_name":"Huaisheng Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Huaisheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140915355","display_name":"Zhengyu Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Zhengyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140926743","display_name":"Yige Yuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan, Yige","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140856804","display_name":"Zhengyu Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Zhengyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140947764","display_name":"Shakti Senthil","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Senthil, Shakti","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140852048","display_name":"Hannaneh Hajishirzi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hajishirzi, Hannaneh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140954235","display_name":"Yulia Tsvetkov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tsvetkov, Yulia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140915303","display_name":"Pradeep Dasigi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dasigi, Pradeep","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5140774326","display_name":"Teng Xiao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao, Teng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.29190000891685486,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.29190000891685486,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12090","display_name":"Language and cultural evolution","score":0.10679999738931656,"subfield":{"id":"https://openalex.org/subfields/3316","display_name":"Cultural Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11975","display_name":"Evolutionary Algorithms and Applications","score":0.10620000213384628,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6766999959945679},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5903000235557556},{"id":"https://openalex.org/keywords/overfitting","display_name":"Overfitting","score":0.5863999724388123},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.5647000074386597},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.5296000242233276},{"id":"https://openalex.org/keywords/sequential-analysis","display_name":"Sequential analysis","score":0.3456999957561493},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.29820001125335693}],"concepts":[{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6766999959945679},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6315000057220459},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5903000235557556},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.5863999724388123},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.5647000074386597},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.5296000242233276},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4449999928474426},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39570000767707825},{"id":"https://openalex.org/C80478641","wikidata":"https://www.wikidata.org/wiki/Q195771","display_name":"Sequential analysis","level":2,"score":0.3456999957561493},{"id":"https://openalex.org/C147297375","wikidata":"https://www.wikidata.org/wiki/Q6674930","display_name":"Look-ahead","level":2,"score":0.29820001125335693},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.29820001125335693},{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.2921000123023987},{"id":"https://openalex.org/C45235069","wikidata":"https://www.wikidata.org/wiki/Q278425","display_name":"Table (database)","level":2,"score":0.28380000591278076},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2685999870300293},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.2623000144958496},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.258899986743927},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.2533999979496002}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.12227","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.12227","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.12227","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.12227","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0],"revisit":[1],"the":[2,30,97,100,104,107,153,190,198],"evaluation":[3,102,125,202],"of":[4,192],"automatic":[5,169,193,207],"harness":[6,12,22,41,88,127,170,194,208],"evolution":[7,13,42,128,171,195],"for":[8,21,200,206],"LLM":[9],"agents.":[10],"Existing":[11],"methods":[14,179],"use":[15],"unit":[16],"test":[17],"cases":[18],"to":[19,80,112,150],"search":[20,47,72,93,98],"configurations":[23],"and":[24,52,77,99,133,139,142,163,180,196,204],"then":[25],"report":[26],"final":[27,101],"performance":[28],"on":[29,147,158],"same":[31,105],"public":[32],"benchmark.":[33],"This":[34],"protocol":[35],"raises":[36],"two":[37],"fundamental":[38],"concerns.":[39],"First,":[40],"is":[43,212],"itself":[44],"an":[45,123],"iterative":[46],"procedure":[48],"that":[49,113,168],"repeatedly":[50],"evaluates":[51],"revises":[53],"candidate":[54],"harnesses":[55,146],"using":[56],"task":[57,115],"feedback.":[58],"As":[59],"in":[60],"agentic":[61],"test-time":[62,131,177],"scaling,":[63],"it":[64],"should":[65],"therefore":[66],"be":[67],"compared":[68],"with":[69,129,161],"simple":[70,130,176],"task-level":[71],"baselines":[73,135],"under":[74,136],"matched":[75],"feedback":[76,138],"inference":[78,140],"budgets":[79],"determine":[81],"whether":[82,152],"its":[83],"gains":[84,109],"arise":[85],"from":[86,91],"improved":[87],"design":[89],"or":[90],"additional":[92],"alone.":[94],"Second,":[95],"because":[96],"share":[103],"benchmark,":[106],"reported":[108],"risk":[110],"overfitting":[111],"specific":[114],"set.":[116],"To":[117],"address":[118],"these":[119],"concerns,":[120],"we":[121],"conduct":[122],"extensive":[124],"comparing":[126],"scaling":[132,178],"discovery":[134],"comparable":[137],"budgets,":[141],"also":[143],"evaluate":[144],"evolved":[145],"held-out":[148],"tasks":[149],"assess":[151],"discovered":[154],"improvements":[155],"generalize.":[156],"Experiments":[157],"Terminal-Bench":[159],"2.1":[160],"GPT-5.4":[162],"Claude":[164],"Opus":[165],"4.6":[166],"show":[167],"does":[172],"not":[173],"consistently":[174],"outperform":[175],"exhibits":[181],"limited":[182],"generalization.":[183],"Our":[184,210],"results":[185],"raise":[186],"important":[187],"questions":[188],"about":[189],"effectiveness":[191],"highlight":[197],"need":[199],"fairer":[201],"protocols":[203],"benchmarks":[205],"design.":[209],"code":[211],"available":[213],"at":[214],"https://github.com/rethinking-harness-evolution.":[215]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-16T00:00:00"}