{"id":"https://openalex.org/W7162800778","doi":"https://doi.org/10.48550/arxiv.2605.29253","title":"OpenClawBench: Benchmarking Process-side Anomalies in Real-world Agent Execution Trajectories","display_name":"OpenClawBench: Benchmarking Process-side Anomalies in Real-world Agent Execution Trajectories","publication_year":2026,"publication_date":"2026-05-28","ids":{"openalex":"https://openalex.org/W7162800778","doi":"https://doi.org/10.48550/arxiv.2605.29253"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.29253","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.29253","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.29253","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137380593","display_name":"Yibing Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yibing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064773495","display_name":"Yangze Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yangze","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100922863","display_name":"Xiaolong Yin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Xiaolong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137309821","display_name":"Bin Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Bin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137331548","display_name":"Chong Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Chong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137336530","display_name":"Hao Yin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Hao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5086112796","display_name":"Zhongyi Han","orcid":"https://orcid.org/0000-0003-2851-193X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Zhongyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.4830999970436096,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.4830999970436096,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.053300000727176666,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.045099999755620956,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6937999725341797},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6891999840736389},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.6525999903678894},{"id":"https://openalex.org/keywords/oracle","display_name":"Oracle","score":0.6406000256538391},{"id":"https://openalex.org/keywords/anomaly-detection","display_name":"Anomaly detection","score":0.6338000297546387},{"id":"https://openalex.org/keywords/binary-number","display_name":"Binary number","score":0.5123000144958496},{"id":"https://openalex.org/keywords/anomaly","display_name":"Anomaly (physics)","score":0.4934999942779541},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.46869999170303345},{"id":"https://openalex.org/keywords/event","display_name":"Event (particle physics)","score":0.45339998602867126}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7484999895095825},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6937999725341797},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6891999840736389},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.6525999903678894},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.6406000256538391},{"id":"https://openalex.org/C739882","wikidata":"https://www.wikidata.org/wiki/Q3560506","display_name":"Anomaly detection","level":2,"score":0.6338000297546387},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.5123000144958496},{"id":"https://openalex.org/C12997251","wikidata":"https://www.wikidata.org/wiki/Q567560","display_name":"Anomaly (physics)","level":2,"score":0.4934999942779541},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.46869999170303345},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.45339998602867126},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.4431999921798706},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3889000117778778},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.36820000410079956},{"id":"https://openalex.org/C66905080","wikidata":"https://www.wikidata.org/wiki/Q17005494","display_name":"Binary classification","level":3,"score":0.36000001430511475},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.35429999232292175},{"id":"https://openalex.org/C2776544517","wikidata":"https://www.wikidata.org/wiki/Q189447","display_name":"Unexpected events","level":2,"score":0.3197999894618988},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.31630000472068787},{"id":"https://openalex.org/C41550386","wikidata":"https://www.wikidata.org/wiki/Q529909","display_name":"Multi-agent system","level":2,"score":0.30149999260902405},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2939999997615814},{"id":"https://openalex.org/C94915269","wikidata":"https://www.wikidata.org/wiki/Q1834857","display_name":"Detector","level":2,"score":0.28029999136924744},{"id":"https://openalex.org/C175551986","wikidata":"https://www.wikidata.org/wiki/Q47089","display_name":"Fault (geology)","level":2,"score":0.2761000096797943},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.27000001072883606},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.2687000036239624},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.2662999927997589},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.2612999975681305},{"id":"https://openalex.org/C74072328","wikidata":"https://www.wikidata.org/wiki/Q1142726","display_name":"Intelligent agent","level":2,"score":0.25589999556541443},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.25200000405311584}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.29253","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.29253","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.29253","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.29253","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Task":[0],"success":[1],"can":[2],"hide":[3],"process":[4,82],"anomalies":[5,53],"in":[6,54,138],"real-world":[7],"agent":[8,11,56,140,168,183],"executions.":[9,141],"An":[10],"may":[12],"pass":[13],"the":[14,39,86,110,150,159],"final":[15],"task":[16],"oracle":[17],"while":[18],"still":[19,120],"accumulating":[20],"unresolved":[21],"ambiguity,":[22],"unsafe":[23],"external":[24],"writes,":[25],"ignored":[26],"errors,":[27],"weakly":[28],"grounded":[29],"commitments,":[30],"or":[31],"capability-boundary":[32],"overcommitment.":[33],"We":[34],"study":[35],"this":[36],"mismatch":[37],"as":[38],"Outcome-Process":[40,111],"Gap":[41,112],"and":[42,50,71,101,173,179],"introduce":[43],"OpenClawBench,":[44,107],"a":[45,102,132],"large-scale":[46],"dataset":[47],"for":[48,176],"measuring":[49],"supervising":[51],"process-side":[52,136],"real":[55,139,167],"execution":[57,169],"processes.":[58],"OpenClawBench":[59,165],"is":[60],"built":[61],"from":[62],"BFCL-driven":[63],"OpenClaw":[64],"sessions":[65],"produced":[66],"by":[67],"6":[68],"source":[69],"models":[70],"contains":[72],"31,264":[73],"annotated":[74],"trajectories.":[75],"It":[76],"aligns":[77],"task-oracle":[78],"outcomes":[79],"with":[80],"structured":[81,90],"evidence.":[83],"FullTax":[84,152],"converts":[85],"aligned":[87],"trajectories":[88],"into":[89,171],"anomaly":[91,104],"supervision:":[92],"binary":[93,156],"labels,":[94],"supporting":[95],"evidence,":[96],"onset/span":[97],"localization,":[98],"severity,":[99],"recoverability,":[100],"5-class":[103],"taxonomy.":[105],"Using":[106],"we":[108],"make":[109],"measurable.":[113],"Among":[114],"31,135":[115],"oracle-passing":[116],"executions,":[117],"2,904":[118],"are":[119],"labeled":[121],"process-anomalous":[122],"under":[123],"FullTax.":[124],"These":[125],"results":[126],"show":[127],"that":[128],"success-only":[129],"evaluation":[130],"misses":[131],"concrete":[133],"class":[134],"of":[135],"failures":[137],"A":[142],"LoRA-fine-tuned":[143],"Gemma":[144],"3":[145],"12B":[146],"detector":[147],"trained":[148],"on":[149,158],"high-confidence":[151],"supervised":[153],"pool":[154],"reaches":[155],"F1=0.729":[157],"cleaner-labels":[160],"held-out":[161],"test":[162],"split.":[163],"Together,":[164],"turns":[166],"logs":[170],"auditable":[172],"reusable":[174],"supervision":[175],"studying,":[177],"diagnosing,":[178],"operationally":[180],"monitoring":[181],"runtime":[182],"reliability.":[184]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-30T00:00:00"}