{"id":"https://openalex.org/W7161251449","doi":"https://doi.org/10.48550/arxiv.2605.14133","title":"ClawForge: Generating Executable Interactive Benchmarks for Command-Line Agents","display_name":"ClawForge: Generating Executable Interactive Benchmarks for Command-Line Agents","publication_year":2026,"publication_date":"2026-05-13","ids":{"openalex":"https://openalex.org/W7161251449","doi":"https://doi.org/10.48550/arxiv.2605.14133"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.14133","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14133","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.14133","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111304741","display_name":"Yuxiang Lai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lai, Yuxiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102208005","display_name":"Peng Xia","orcid":"https://orcid.org/0009-0006-9190-0208"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xia, Peng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136223546","display_name":"Haonian Ji","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ji, Haonian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136225709","display_name":"Kaiwen Xiong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiong, Kaiwen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125507472","display_name":"Kaide Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Kaide","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136256334","display_name":"Jiaqi Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jiaqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136248715","display_name":"Fang Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Fang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088198600","display_name":"Jike Zhong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhong, Jike","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136242229","display_name":"Zeyu Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Zeyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136232155","display_name":"Cihang Xie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xie, Cihang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136245087","display_name":"Huaxiu Yao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Huaxiu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.1477999985218048,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.1477999985218048,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.08079999685287476,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.05689999833703041,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/executable","display_name":"Executable","score":0.8518000245094299},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.8041999936103821},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7576000094413757},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6421999931335449},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5940999984741211},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.5412999987602234},{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.31779998540878296},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.2994999885559082}],"concepts":[{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.8518000245094299},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.8041999936103821},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7904000282287598},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7576000094413757},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6421999931335449},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5940999984741211},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.5412999987602234},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.38350000977516174},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.38190001249313354},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3352999985218048},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.322299987077713},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.31779998540878296},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3052000105381012},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2994999885559082},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.2976999878883362},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.2962000072002411},{"id":"https://openalex.org/C167822520","wikidata":"https://www.wikidata.org/wiki/Q176452","display_name":"Finite-state machine","level":2,"score":0.28940001130104065},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.2874999940395355},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2648000121116638},{"id":"https://openalex.org/C31266012","wikidata":"https://www.wikidata.org/wiki/Q6554340","display_name":"Linkage (software)","level":3,"score":0.2639999985694885},{"id":"https://openalex.org/C41550386","wikidata":"https://www.wikidata.org/wiki/Q529909","display_name":"Multi-agent system","level":2,"score":0.263700008392334},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.26019999384880066},{"id":"https://openalex.org/C80519477","wikidata":"https://www.wikidata.org/wiki/Q3532236","display_name":"Scenario testing","level":3,"score":0.2540000081062317},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.251800000667572}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.14133","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14133","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.14133","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14133","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Interactive":[0],"agent":[1,41],"benchmarks":[2,38],"face":[3],"a":[4,68],"tension":[5],"between":[6],"scalable":[7],"construction":[8],"and":[9,19,51,90,96,110,154,173,188],"realistic":[10],"workflow":[11,104],"evaluation.":[12],"Hand-authored":[13],"tasks":[14,47],"are":[15,181],"expensive":[16],"to":[17,160],"extend":[18],"revise,":[20],"while":[21],"static":[22],"prompt":[23],"evaluation":[24,42],"misses":[25],"failures":[26,180],"that":[27,137,178,189],"only":[28,142],"appear":[29],"when":[30],"agents":[31,57,98,166],"operate":[32],"over":[33,102],"persistent":[34,103],"state.":[35],"Existing":[36],"interactive":[37],"have":[39],"advanced":[40],"significantly,":[43],"but":[44],"most":[45],"initialize":[46],"from":[48],"clean":[49],"state":[50,77,109,169,197],"do":[52],"not":[53],"systematically":[54],"test":[55],"how":[56],"handle":[58],"pre-existing":[59],"partial,":[60],"stale,":[61],"or":[62],"conflicting":[63],"artifacts.":[64],"We":[65,119],"present":[66],"\\textbf{ClawForge},":[67],"generator-backed":[69],"benchmark":[70],"framework":[71,80,122],"for":[72,151],"executable":[73],"command-line":[74],"workflows":[75],"under":[76,196],"conflict.":[78,198],"The":[79],"compiles":[81],"scenario":[82],"templates,":[83],"grounded":[84],"slots,":[85],"initialized":[86],"state,":[87],"reference":[88],"trajectories,":[89],"validators":[91],"into":[92],"reproducible":[93],"task":[94],"specifications,":[95],"evaluates":[97],"step":[99,101],"by":[100,164],"surfaces":[105],"using":[106],"normalized":[107],"end":[108],"observable":[111],"side":[112],"effects":[113],"rather":[114,184],"than":[115,185],"exact":[116],"trajectory":[117],"matching.":[118],"instantiate":[120],"this":[121],"as":[123],"the":[124,138,155],"ClawForge-Bench":[125],"(17":[126],"scenarios,":[127],"6":[128],"ability":[129],"categories).":[130],"Results":[131],"across":[132],"seven":[133],"frontier":[134],"models":[135,190],"show":[136],"best":[139],"model":[140,157],"reaches":[141],"45.3%":[143],"strict":[144],"accuracy,":[145],"wrong-state":[146],"replacement":[147],"remains":[148],"below":[149],"17\\%":[150],"all":[152],"models,":[153],"widest":[156],"separation":[158],"(17%":[159],"90%)":[161],"is":[162],"driven":[163],"whether":[165],"inspect":[167],"existing":[168],"before":[170],"acting.":[171],"Partial-credit":[172],"step-efficiency":[174],"analyses":[175],"further":[176],"reveal":[177],"many":[179],"near-miss":[182],"closures":[183],"early":[186],"breakdowns,":[187],"exhibit":[191],"qualitatively":[192],"different":[193],"failure":[194],"styles":[195]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-16T00:00:00"}