{"id":"https://openalex.org/W7160895932","doi":"https://doi.org/10.48550/arxiv.2605.10597","title":"CrackMeBench: Binary Reverse Engineering for Agents","display_name":"CrackMeBench: Binary Reverse Engineering for Agents","publication_year":2026,"publication_date":"2026-05-11","ids":{"openalex":"https://openalex.org/W7160895932","doi":"https://doi.org/10.48550/arxiv.2605.10597"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.10597","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.10597","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.10597","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135926837","display_name":"Isaac David","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"David, Isaac","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5063253761","display_name":"Arthur Gervais","orcid":"https://orcid.org/0000-0002-3565-3410"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gervais, Arthur","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.39570000767707825,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.39570000767707825,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.26460000872612,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10126","display_name":"Logic, programming, and type systems","score":0.07169999927282333,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/testbed","display_name":"Testbed","score":0.675000011920929},{"id":"https://openalex.org/keywords/executable","display_name":"Executable","score":0.6417999863624573},{"id":"https://openalex.org/keywords/reverse-engineering","display_name":"Reverse engineering","score":0.6305999755859375},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6078000068664551},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.5515000224113464},{"id":"https://openalex.org/keywords/binary-number","display_name":"Binary number","score":0.5351999998092651},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5177000164985657},{"id":"https://openalex.org/keywords/tracing","display_name":"Tracing","score":0.4544000029563904},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.435699999332428},{"id":"https://openalex.org/keywords/calibration","display_name":"Calibration","score":0.4043000042438507}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.695900022983551},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.675000011920929},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.6417999863624573},{"id":"https://openalex.org/C207850805","wikidata":"https://www.wikidata.org/wiki/Q269608","display_name":"Reverse engineering","level":2,"score":0.6305999755859375},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6078000068664551},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5515000224113464},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.5351999998092651},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5177000164985657},{"id":"https://openalex.org/C138673069","wikidata":"https://www.wikidata.org/wiki/Q322229","display_name":"Tracing","level":2,"score":0.4544000029563904},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.435699999332428},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.4043000042438507},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.38670000433921814},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.3799999952316284},{"id":"https://openalex.org/C169590947","wikidata":"https://www.wikidata.org/wiki/Q47506","display_name":"Compiler","level":2,"score":0.3555000126361847},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3361999988555908},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.32249999046325684},{"id":"https://openalex.org/C92446256","wikidata":"https://www.wikidata.org/wiki/Q3306762","display_name":"Data validation","level":2,"score":0.3215000033378601},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.31839999556541443},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3107999861240387},{"id":"https://openalex.org/C2779190172","wikidata":"https://www.wikidata.org/wiki/Q4913888","display_name":"Binary data","level":3,"score":0.30880001187324524},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.3084000051021576},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.304500013589859},{"id":"https://openalex.org/C2780992000","wikidata":"https://www.wikidata.org/wiki/Q17016113","display_name":"Generator (circuit theory)","level":3,"score":0.30399999022483826},{"id":"https://openalex.org/C112968700","wikidata":"https://www.wikidata.org/wiki/Q11368","display_name":"Unix","level":3,"score":0.3028999865055084},{"id":"https://openalex.org/C63435697","wikidata":"https://www.wikidata.org/wiki/Q864135","display_name":"Binary code","level":3,"score":0.2985000014305115},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2903999984264374},{"id":"https://openalex.org/C2779639559","wikidata":"https://www.wikidata.org/wiki/Q7661178","display_name":"Symbolic execution","level":3,"score":0.28540000319480896},{"id":"https://openalex.org/C2779803651","wikidata":"https://www.wikidata.org/wiki/Q5282088","display_name":"Discriminator","level":3,"score":0.2799000144004822},{"id":"https://openalex.org/C167981075","wikidata":"https://www.wikidata.org/wiki/Q2667186","display_name":"Sandbox (software development)","level":2,"score":0.27459999918937683},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.274399995803833},{"id":"https://openalex.org/C2781052500","wikidata":"https://www.wikidata.org/wiki/Q2230313","display_name":"Shell (structure)","level":2,"score":0.2721000015735626},{"id":"https://openalex.org/C162319229","wikidata":"https://www.wikidata.org/wiki/Q175263","display_name":"Data structure","level":2,"score":0.2632000148296356},{"id":"https://openalex.org/C2780069185","wikidata":"https://www.wikidata.org/wiki/Q7977945","display_name":"Equivalence (formal languages)","level":2,"score":0.25760000944137573},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.25209999084472656},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.25049999356269836}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.10597","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.10597","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.10597","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.10597","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"No poverty","id":"https://metadata.un.org/sdg/1","score":0.615942656993866}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Benchmarks":[0],"for":[1,53,148,152,159,217],"coding":[2],"agents":[3,56,108],"increasingly":[4,12],"measure":[5,13],"source-level":[6],"software":[7],"repair,":[8],"and":[9,35,78,104,107,133,156,176,189,194,209],"cybersecurity":[10],"benchmarks":[11],"broad":[14],"capture-the-flag":[15],"performance.":[16],"Classical":[17],"binary":[18,66,225],"reverse":[19],"engineering":[20],"remains":[21],"less":[22],"precisely":[23],"specified:":[24],"given":[25],"only":[26],"an":[27,30,37,111],"executable,":[28],"can":[29],"agent":[31],"recover":[32],"validation":[33,67],"logic":[34],"produce":[36],"input,":[38],"serial,":[39],"artifact,":[40],"or":[41],"key":[42],"generator":[43],"accepted":[44],"by":[45],"the":[46,141,167,180],"program?":[47],"We":[48],"introduce":[49],"CrackMeBench,":[50],"a":[51,116,126,130,214],"benchmark":[52,88],"evaluating":[54],"language-model":[55],"on":[57,64,140,179],"educational":[58],"CrackMe-style":[59],"reverse-engineering":[60,123],"tasks.":[61],"CrackMeBench":[62,191],"focuses":[63],"deterministic":[65],"problems":[68],"with":[69,94,121,129,171],"executable":[70],"oracles,":[71],"symbol-poor":[72],"binaries,":[73],"explicit":[74],"local":[75],"tool":[76,202],"access,":[77],"externally":[79],"scored":[80,135,196],"submissions":[81,136],"rather":[82],"than":[83],"free-form":[84],"explanations.":[85],"The":[86,162],"v0":[87],"combines":[89],"eight":[90],"public":[91,182],"calibration":[92,183],"CrackMes":[93],"twelve":[95],"generated":[96,142,164],"main-score":[97],"tasks":[98,146],"built":[99],"from":[100,220],"seeded":[101],"C,":[102],"Rust,":[103],"Go":[105],"templates,":[106],"run":[109],"through":[110],"equal":[112],"shell":[113],"interface":[114],"in":[115],"no-network":[117],"Linux":[118],"Docker":[119],"sandbox":[120],"standard":[122],"tools.":[124],"In":[125],"three-model":[127],"evaluation":[128],"five-minute":[131],"budget":[132],"three":[134],"per":[137],"task,":[138],"pass@3":[139,172,185],"split":[143],"is":[144,186],"11/12":[145],"(92%)":[147],"GPT-5.5,":[149],"7/12":[150],"(58%)":[151],"Claude":[153],"Opus":[154],"4.7,":[155],"5/12":[157],"(42%)":[158],"Kimi":[160],"K2.":[161],"harder":[163],"half":[165],"separates":[166],"models":[168],"more":[169],"sharply,":[170],"of":[173],"5/6,":[174],"2/6,":[175],"1/6,":[177],"respectively;":[178],"eight-task":[181],"split,":[184],"3/8,":[187],"2/8,":[188],"1/8.":[190],"records":[192],"pass@1":[193],"pass@3,":[195],"submissions,":[197],"wall-clock":[198],"time,":[199],"command":[200],"traces,":[201],"categories,":[203],"provider-reported":[204],"token":[205],"usage,":[206],"estimated":[207],"cost,":[208],"qualitative":[210],"failure":[211],"labels,":[212],"providing":[213],"reproducible":[215],"testbed":[216],"measuring":[218],"progress":[219],"source-code":[221],"reasoning":[222],"toward":[223],"autonomous":[224],"analysis":[226],"while":[227],"restricting":[228],"scope":[229],"to":[230],"educational,":[231],"purpose-built":[232],"programs.":[233]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-13T00:00:00"}