{"id":"https://openalex.org/W7162002888","doi":"https://doi.org/10.48550/arxiv.2605.21384","title":"SpecBench: Measuring Reward Hacking in Long-Horizon Coding Agents","display_name":"SpecBench: Measuring Reward Hacking in Long-Horizon Coding Agents","publication_year":2026,"publication_date":"2026-05-20","ids":{"openalex":"https://openalex.org/W7162002888","doi":"https://doi.org/10.48550/arxiv.2605.21384"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.21384","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.21384","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.21384","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136617392","display_name":"Bingchen Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Bingchen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136668553","display_name":"Dhruv Srikanth","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Srikanth, Dhruv","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136708816","display_name":"Yuxiang Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Yuxiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136697255","display_name":"Zhengyao Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Zhengyao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.48089998960494995,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.48089998960494995,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.1469999998807907,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10126","display_name":"Logic, programming, and type systems","score":0.12780000269412994,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/testbed","display_name":"Testbed","score":0.5845999717712402},{"id":"https://openalex.org/keywords/javascript","display_name":"JavaScript","score":0.5666999816894531},{"id":"https://openalex.org/keywords/hacker","display_name":"Hacker","score":0.5604000091552734},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.5455999970436096},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.413100004196167},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.3946000039577484},{"id":"https://openalex.org/keywords/source-lines-of-code","display_name":"Source lines of code","score":0.37540000677108765},{"id":"https://openalex.org/keywords/parsing","display_name":"Parsing","score":0.3734000027179718},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.3686999976634979}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7663999795913696},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.5845999717712402},{"id":"https://openalex.org/C544833334","wikidata":"https://www.wikidata.org/wiki/Q2005","display_name":"JavaScript","level":2,"score":0.5666999816894531},{"id":"https://openalex.org/C86844869","wikidata":"https://www.wikidata.org/wiki/Q2798820","display_name":"Hacker","level":2,"score":0.5604000091552734},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.5455999970436096},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4523000121116638},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.413100004196167},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4000999927520752},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.3946000039577484},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.37540000677108765},{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.3734000027179718},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.3686999976634979},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3467000126838684},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33809998631477356},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.33799999952316284},{"id":"https://openalex.org/C51929080","wikidata":"https://www.wikidata.org/wiki/Q2425187","display_name":"Codebase","level":3,"score":0.3345000147819519},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.3280999958515167},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3147999942302704},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.31459999084472656},{"id":"https://openalex.org/C163339463","wikidata":"https://www.wikidata.org/wiki/Q484598","display_name":"CAPTCHA","level":2,"score":0.30660000443458557},{"id":"https://openalex.org/C76844732","wikidata":"https://www.wikidata.org/wiki/Q4072285","display_name":"Conformance testing","level":3,"score":0.3059000074863434},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.30230000615119934},{"id":"https://openalex.org/C169903001","wikidata":"https://www.wikidata.org/wiki/Q3264987","display_name":"Reciprocity (cultural anthropology)","level":2,"score":0.28940001130104065},{"id":"https://openalex.org/C577917","wikidata":"https://www.wikidata.org/wiki/Q189223","display_name":"Turing test","level":2,"score":0.28119999170303345},{"id":"https://openalex.org/C2775941552","wikidata":"https://www.wikidata.org/wiki/Q25212305","display_name":"Isolation (microbiology)","level":2,"score":0.2777999937534332},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.2775999903678894},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.2736999988555908},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2727999985218048},{"id":"https://openalex.org/C106251023","wikidata":"https://www.wikidata.org/wiki/Q851989","display_name":"Porting","level":3,"score":0.2680000066757202},{"id":"https://openalex.org/C2779785983","wikidata":"https://www.wikidata.org/wiki/Q5957248","display_name":"Hyperbolic discounting","level":3,"score":0.25690001249313354},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.2556000053882599},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.2542000114917755}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.21384","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.21384","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.21384","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.21384","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0],"long-horizon":[1],"coding":[2,243],"agents":[3,244],"produce":[4],"more":[5],"code":[6,215],"than":[7],"any":[8],"developer":[9],"can":[10,109],"review,":[11],"oversight":[12],"collapses":[13],"onto":[14],"a":[15,58,98,106,140,154,173,227,237],"single":[16],"surface:":[17],"the":[18,30,39,63,90,93,114,120,181,252],"automated":[19],"test":[20,96,233,253],"suite.":[21],"Reward":[22],"hacking":[23,47,185],"naturally":[24],"arises":[25],"in":[26,73,122,214],"this":[27,45,135],"setup,":[28],"as":[29],"agent":[31,100,179],"optimizes":[32],"for":[33,210,240],"passing":[34],"tests":[35,68,78],"while":[36,176],"deviating":[37],"from":[38,148,168,219],"users":[40],"true":[41],"goal.":[42],"We":[43],"study":[44],"reward":[46,131,184],"phenomenon":[48],"by":[49,206],"decompose":[50],"software":[51],"engineering":[52],"tasks":[53,146,151,161],"into":[54],"three":[55],"parts:":[56],"(i)":[57],"natural":[59],"language":[60],"description":[61],"of":[62,113],"specification":[64,91],"(ii)":[65],"visible":[66,94,182],"validation":[67,95],"that":[69,79,108,231],"exercise":[70],"specified":[71],"features":[72,83],"isolation,":[74],"and":[75,92],"(iii)":[76],"held-out":[77,115],"compose":[80],"those":[81],"same":[82],"to":[84,104,129,157,223],"simulate":[85],"real-world":[86],"usage.":[87],"Based":[88,133],"on":[89,125,134,193],"suites,":[97],"genuine":[99,246],"would":[101],"be":[102],"able":[103],"generate":[105],"solution":[107],"also":[110,198],"pass":[111,123],"all":[112],"tests.":[116],"Therefore":[117],"we":[118,137],"use":[119],"gap":[121,197],"rates":[124],"these":[126],"two":[127],"suites":[128,254],"quantify":[130],"hacking.":[132],"methodology,":[136],"introduce":[138],"SpecBench,":[139],"benchmark":[141],"comprising":[142],"30":[143],"systems-level":[144],"programming":[145],"ranging":[147],"short":[149],"horizon":[150,160],"like":[152,162],"building":[153,163],"JSON":[155],"parser":[156],"ultra":[158],"long":[159],"an":[164],"entire":[165],"OS":[166],"kernel":[167],"scratch.":[169],"Large-scale":[170],"experiments":[171],"reveal":[172],"consistent":[174],"pattern:":[175],"every":[177,211],"frontier":[178],"saturates":[180],"suite,":[183],"persists,":[186],"with":[187,201],"smaller":[188],"models":[189],"exhibiting":[190],"larger":[191],"gaps":[192],"holdout":[194],"suites.":[195],"The":[196],"scales":[199],"sharply":[200],"task":[202],"length:":[203],"it":[204],"grows":[205],"28":[207],"percentage":[208],"points":[209],"tenfold":[212],"increase":[213],"size.":[216],"Failures":[217],"range":[218],"subtle":[220],"feature":[221],"isolation":[222],"deliberate":[224],"exploits,":[225],"including":[226],"2,900-line":[228],"hash-table":[229],"\"compiler\"":[230],"memorizes":[232],"inputs.":[234],"SpecBench":[235],"offers":[236],"principled":[238],"testbed":[239],"measuring":[241],"whether":[242],"build":[245],"working":[247],"systems":[248],"or":[249],"merely":[250],"game":[251],"developers":[255],"hand":[256],"them.":[257]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-22T00:00:00"}