{"id":"https://openalex.org/W7164437655","doi":"https://doi.org/10.48550/arxiv.2606.12344","title":"Claw-SWE-Bench: A Benchmark for Evaluating OpenClaw-style Agent Harnesses on Coding Tasks","display_name":"Claw-SWE-Bench: A Benchmark for Evaluating OpenClaw-style Agent Harnesses on Coding Tasks","publication_year":2026,"publication_date":"2026-06-10","ids":{"openalex":"https://openalex.org/W7164437655","doi":"https://doi.org/10.48550/arxiv.2606.12344"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.12344","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12344","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.12344","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138450223","display_name":"Mengyu Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Mengyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138478630","display_name":"Kai Han","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Kai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138427464","display_name":"Boxun Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Boxun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138433333","display_name":"Haiyang Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Haiyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127789055","display_name":"Yuchuan Tian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tian, Yuchuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138453039","display_name":"Wei He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138410889","display_name":"Hang Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Hang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138400022","display_name":"Jianyuan Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Jianyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122757499","display_name":"Hailin Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Hailin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138438351","display_name":"Lin Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Lin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138432321","display_name":"Chao Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Chao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138445497","display_name":"Guohao Dai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dai, Guohao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008190519","display_name":"Lixue Xia","orcid":"https://orcid.org/0000-0002-7731-7028"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xia, Lixue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138416108","display_name":"Yunchao Wei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Yunchao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138442725","display_name":"Yunhe Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yunhe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138441839","display_name":"Yu Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.179299995303154,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.179299995303154,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.08669999986886978,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.0617000013589859,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/adapter","display_name":"Adapter (computing)","score":0.8860999941825867},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7458999752998352},{"id":"https://openalex.org/keywords/workspace","display_name":"Workspace","score":0.5633000135421753},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.5550000071525574},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.3280999958515167},{"id":"https://openalex.org/keywords/calibration","display_name":"Calibration","score":0.325300008058548}],"concepts":[{"id":"https://openalex.org/C177284502","wikidata":"https://www.wikidata.org/wiki/Q1005390","display_name":"Adapter (computing)","level":2,"score":0.8860999941825867},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7458999752998352},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7444999814033508},{"id":"https://openalex.org/C58581272","wikidata":"https://www.wikidata.org/wiki/Q12741163","display_name":"Workspace","level":3,"score":0.5633000135421753},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.5550000071525574},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.3280999958515167},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3276999890804291},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.325300008058548},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.3165999948978424},{"id":"https://openalex.org/C207850805","wikidata":"https://www.wikidata.org/wiki/Q269608","display_name":"Reverse engineering","level":2,"score":0.3109000027179718},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2928999960422516},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.2897000014781952},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.2863999903202057},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.2847000062465668},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.2563999891281128},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.25600001215934753}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.12344","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12344","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.12344","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12344","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"General-purpose":[0],"agents":[1],"such":[2],"as":[3,8,207],"OpenClaw":[4,124,163],"are":[5],"increasingly":[6],"used":[7],"autonomous":[9],"tool":[10],"users,":[11],"but":[12],"their":[13],"coding":[14,158],"ability":[15],"is":[16,106,150,229],"difficult":[17],"to":[18,156],"measure":[19],"under":[20,59,186],"SWE-bench:":[21],"a":[22,44,63,112,126,168,216,220],"generic":[23],"agent":[24,54],"does":[25],"not":[26],"by":[27,111,177,183],"itself":[28],"satisfy":[29],"the":[30,121,135,141],"clean":[31],"Docker":[32],"workspace,":[33],"patch,":[34],"and":[35,48,73,86,92,167,180,204,219,233],"prediction":[36],"contract":[37],"required":[38],"for":[39,102,152,224],"scoring.":[40],"We":[41,97],"introduce":[42],"Claw-SWE-Bench,":[43],"multilingual":[45],"SWE-bench-style":[46],"benchmark":[47,77,218],"adapter":[49,129,137,148],"protocol":[50],"that":[51,147],"makes":[52],"heterogeneous":[53],"harnesses,":[55],"or":[56],"claws,":[57],"comparable":[58],"fair":[60],"settings":[61],"including":[62],"fixed":[64,187],"prompt,":[65],"runtime":[66],"budget,":[67],"workspace":[68],"contract,":[69],"patch":[70],"extraction":[71],"procedure,":[72],"evaluator.":[74],"The":[75,227],"full":[76,122,136,217],"contains":[78],"350":[79],"GitHub":[80],"issue-resolution":[81],"instances":[82],"across":[83],"8":[84],"languages":[85],"43":[87],"repositories,":[88],"drawn":[89],"from":[90],"SWE-bench-Multilingual":[91],"SWE-bench-Verified-Mini":[93],"after":[94],"future-commit":[95],"cleanup.":[96],"also":[98],"release":[99],"Claw-SWE-Bench":[100,200],"Lite":[101],"faster":[103],"validation,":[104],"which":[105],"an":[107,162],"80-instance":[108],"subset":[109],"selected":[110],"cost-aware,":[113],"rank-aware":[114],"procedure":[115],"over":[116],"17":[117],"calibration":[118],"columns.":[119],"On":[120],"benchmark,":[123],"with":[125,140,190],"minimal":[127],"direct-diff":[128],"scores":[130],"only":[131],"$19.1\\%$":[132],"Pass@1,":[133],"whereas":[134],"reaches":[138],"$73.4\\%$":[139],"same":[142],"GLM":[143],"5.1":[144],"backbone,":[145],"showing":[146],"design":[149],"essential":[151],"enabling":[153],"OpenClaw-style":[154],"harnesses":[155],"perform":[157],"tasks":[159],"effectively.":[160],"Across":[161],"$\\times$":[164,170],"nine-model":[165],"sweep":[166],"five-claw":[169],"two-model":[171],"sweep,":[172],"model":[173],"choice":[174,182],"changes":[175],"Pass@1":[176],"$29.4$":[178],"pp":[179,185],"harness":[181,203],"$27.4$":[184],"models;":[188],"systems":[189],"similar":[191],"accuracy":[192],"can":[193],"differ":[194],"substantially":[195],"in":[196],"total":[197],"API":[198],"cost.":[199],"therefore":[201],"treats":[202],"cost":[205],"accounting":[206],"first-class":[208],"axes":[209],"of":[210],"SWE-style":[211],"coding-agent":[212],"evaluation,":[213],"providing":[214],"both":[215],"low-cost":[221],"reference":[222],"set":[223],"reproducible":[225],"comparison.":[226],"data":[228],"available":[230],"at":[231],"https://github.com/opensquilla/claw-swe-bench":[232],"https://huggingface.co/datasets/TokenRhythm/Claw-SWE-Bench.":[234]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-12T00:00:00"}