{"id":"https://openalex.org/W7164039518","doi":"https://doi.org/10.48550/arxiv.2606.08960","title":"Hardening Agent Benchmarks with Adversarial Hacker-Fixer Loops","display_name":"Hardening Agent Benchmarks with Adversarial Hacker-Fixer Loops","publication_year":2026,"publication_date":"2026-06-08","ids":{"openalex":"https://openalex.org/W7164039518","doi":"https://doi.org/10.48550/arxiv.2606.08960"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.08960","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08960","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.08960","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138261073","display_name":"Ziqian Zhong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhong, Ziqian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138233475","display_name":"Ivgeni Segal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Segal, Ivgeni","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123523608","display_name":"Ivan Bercovich","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bercovich, Ivan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120404554","display_name":"Shashwat Saxena","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Saxena, Shashwat","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138224787","display_name":"Kexun Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Kexun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138204624","display_name":"Aditi Raghunathan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Raghunathan, Aditi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.4584999978542328,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.4584999978542328,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.27079999446868896,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12479","display_name":"Web Application Security Vulnerabilities","score":0.06390000134706497,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.7409999966621399},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.5960999727249146},{"id":"https://openalex.org/keywords/loop","display_name":"Loop (graph theory)","score":0.4065999984741211},{"id":"https://openalex.org/keywords/solver","display_name":"Solver","score":0.36309999227523804},{"id":"https://openalex.org/keywords/commit","display_name":"Commit","score":0.3621000051498413},{"id":"https://openalex.org/keywords/hacker","display_name":"Hacker","score":0.35670000314712524},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.34610000252723694}],"concepts":[{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.7409999966621399},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7321000099182129},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.5960999727249146},{"id":"https://openalex.org/C184670325","wikidata":"https://www.wikidata.org/wiki/Q512604","display_name":"Loop (graph theory)","level":2,"score":0.4065999984741211},{"id":"https://openalex.org/C2778770139","wikidata":"https://www.wikidata.org/wiki/Q1966904","display_name":"Solver","level":2,"score":0.36309999227523804},{"id":"https://openalex.org/C153180980","wikidata":"https://www.wikidata.org/wiki/Q19776675","display_name":"Commit","level":2,"score":0.3621000051498413},{"id":"https://openalex.org/C86844869","wikidata":"https://www.wikidata.org/wiki/Q2798820","display_name":"Hacker","level":2,"score":0.35670000314712524},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.34610000252723694},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.32820001244544983},{"id":"https://openalex.org/C164155591","wikidata":"https://www.wikidata.org/wiki/Q2067766","display_name":"Satisfiability modulo theories","level":2,"score":0.32749998569488525},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.29679998755455017},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2879999876022339},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.2874000072479248},{"id":"https://openalex.org/C29122968","wikidata":"https://www.wikidata.org/wiki/Q1414816","display_name":"Incentive","level":2,"score":0.2770000100135803},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.2678000032901764},{"id":"https://openalex.org/C2780643141","wikidata":"https://www.wikidata.org/wiki/Q819049","display_name":"Pledge","level":2,"score":0.2653999924659729},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.2653000056743622},{"id":"https://openalex.org/C55282118","wikidata":"https://www.wikidata.org/wiki/Q252683","display_name":"Snapshot (computer storage)","level":2,"score":0.26109999418258667}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.08960","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08960","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.08960","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08960","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.6374326348304749,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Agent":[0],"benchmarks":[1,26],"score":[2],"submissions":[3],"with":[4],"outcome":[5],"verifiers":[6,67],"that":[7,166],"are":[8],"typically":[9],"hand-written":[10],"and":[11,27,45,55,99,129,188,197,203,243],"brittle,":[12],"leaving":[13],"them":[14],"open":[15],"to":[16,81,94,135,153,199,209],"reward":[17],"hacking.":[18],"We":[19,57,124,163,217],"audit":[20],"1,968":[21],"tasks":[22],"across":[23,133,214],"five":[24],"terminal-agent":[25],"find":[28,165],"323":[29],"(16%)":[30],"hackable":[31,222],"by":[32],"frontier":[33],"models":[34],"given":[35],"only":[36],"the":[37,50,59,83,87,92,103,117,121,137,139,144,147,170,183,231,238,240],"task":[38],"description.":[39],"This":[40],"corrupts":[41],"both":[42],"leaderboard":[43],"rankings":[44],"RL":[46],"training":[47],"signal,":[48],"yet":[49],"standard":[51],"response":[52],"is":[53],"manual":[54,70],"reactive.":[56],"introduce":[58],"hacker-fixer":[60],"loop,":[61],"a":[62,78,89,100,156,228,247],"method":[63],"for":[64,249],"building":[65],"exploit-resistant":[66],"without":[68,85],"per-task":[69],"patching.":[71],"The":[72,110],"loop":[73,111,140,145,171,181,241],"alternates":[74],"three":[75],"LLM":[76],"agents:":[77],"hacker":[79],"tries":[80],"pass":[82],"verifier":[84,93,105,118,127],"solving":[86],"task,":[88],"fixer":[90],"patches":[91,131],"reject":[95],"each":[96,113],"discovered":[97],"exploit,":[98],"solver":[101],"confirms":[102],"patched":[104,236],"still":[106],"admits":[107],"legitimate":[108],"solutions.":[109],"iterates:":[112],"patch":[114],"reshapes":[115],"what":[116],"rewards,":[119],"surfacing":[120],"next":[122],"exploit.":[123],"further":[125],"add":[126],"access,":[128],"let":[130],"transfer":[132],"tasks,":[134],"broaden":[136],"exploits":[138,239],"discovers.":[141],"On":[142],"KernelBench,":[143,202],"drives":[146,182],"attack":[148,192,233],"success":[149,193],"rate":[150,194],"from":[151,195,207],"62%":[152],"0%":[154,200],"on":[155,201,211],"held-out":[157],"corpus":[158],"of":[159,230],"publicly":[160],"reported":[161],"exploits.":[162],"also":[164],"weaker":[167],"agents":[168],"in":[169],"can":[172],"defend":[173],"against":[174],"much":[175],"stronger":[176,184],"hackers:":[177],"Gemini":[178,185,204],"3":[179],"Flash's":[180],"3.1":[186,205],"Pro":[187],"Claude":[189],"Opus":[190],"4.7's":[191],"76%":[196],"61%":[198],"Pro's":[206],"39%":[208],"17%":[210],"Terminal":[212,219],"Bench":[213],"77":[215],"tasks.":[216],"release":[218],"Wrench":[220],"(323":[221],"environments,":[223],"3,632":[224],"hack":[225],"trajectories)":[226],"as":[227,246],"snapshot":[229],"current":[232],"surface,":[234],"our":[235,244],"verifiers,":[237],"discovered,":[242],"implementation":[245],"basis":[248],"future":[250],"work.":[251]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-10T00:00:00"}