{"id":"https://openalex.org/W7171976702","doi":"https://doi.org/10.48550/arxiv.2607.28037","title":"ClawTrack: Towards Trace-Level Evaluation and Improvement of Real-World Autonomous Agents","display_name":"ClawTrack: Towards Trace-Level Evaluation and Improvement of Real-World Autonomous Agents","publication_year":2026,"publication_date":"2026-07-30","ids":{"openalex":"https://openalex.org/W7171976702","doi":"https://doi.org/10.48550/arxiv.2607.28037"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.28037","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.28037","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.28037","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5144137101","display_name":"Xingjian Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Xingjian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135735089","display_name":"Xuhang Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Xuhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5144125341","display_name":"Xingchen Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xingchen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5144098350","display_name":"Junlin Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Junlin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5144130791","display_name":"Jianing Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jianing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5144172738","display_name":"Linsen Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Linsen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5144125913","display_name":"Xiaoyu Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Xiaoyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5144152428","display_name":"Xuezhi Cao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Xuezhi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5144150672","display_name":"Xunliang Cai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cai, Xunliang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.21819999814033508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.21819999814033508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.09570000320672989,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.07800000160932541,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.7099000215530396},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6833999752998352},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6385999917984009},{"id":"https://openalex.org/keywords/counterfactual-thinking","display_name":"Counterfactual thinking","score":0.3917999863624573},{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.3723999857902527},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.3628999888896942},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.3490000069141388}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.7099000215530396},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6833999752998352},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6385999917984009},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6304000020027161},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5965999960899353},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4771000146865845},{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.3917999863624573},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.3723999857902527},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.3628999888896942},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.3490000069141388},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.30640000104904175},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3012000024318695},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2996000051498413},{"id":"https://openalex.org/C20162079","wikidata":"https://www.wikidata.org/wiki/Q1151406","display_name":"Case-based reasoning","level":2,"score":0.2777000069618225},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.27630001306533813},{"id":"https://openalex.org/C143299363","wikidata":"https://www.wikidata.org/wiki/Q900584","display_name":"Attribution","level":2,"score":0.26820001006126404},{"id":"https://openalex.org/C195344581","wikidata":"https://www.wikidata.org/wiki/Q2555318","display_name":"Automated reasoning","level":2,"score":0.2644999921321869},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.2540999948978424}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.28037","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.28037","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.28037","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.28037","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0],"LLM-based":[1],"agents":[2],"are":[3,135],"deployed":[4],"in":[5,39],"complex,":[6],"multi-step":[7],"workflows,":[8],"a":[9,48],"critical":[10],"evaluation":[11],"gap":[12],"has":[13],"emerged:":[14],"most":[15],"existing":[16],"benchmarks":[17],"judge":[18,154],"only":[19],"final":[20],"outcomes,":[21],"unable":[22],"to":[23,33,120,128,149],"distinguish":[24],"reliable":[25],"reasoning":[26,84,122],"from":[27],"lucky":[28,125],"success":[29,117],"or":[30],"attribute":[31,116],"failures":[32],"specific":[34,121],"process":[35,113],"deficiencies,":[36],"hindering":[37],"attribution":[38],"long-horizon":[40],"tasks.":[41],"In":[42],"this":[43],"work,":[44],"we":[45,109],"present":[46],"ClawTrack,":[47],"dual-assessment":[49],"benchmark":[50],"that":[51],"simultaneously":[52],"measures":[53],"what":[54],"an":[55],"agent":[56],"achieves":[57,63],"(Task":[58],"Score)":[59],"and":[60,94,118,156],"how":[61],"it":[62,64],"(Process":[65],"Score).":[66],"ClawTrack":[67],"comprises":[68],"320":[69],"tasks":[70],"across":[71,152,165],"8":[72],"domains":[73],"with":[74,137],"25+":[75],"deterministic":[76],"mock":[77],"services.":[78],"A":[79],"Process":[80],"Grader":[81],"scores":[82,114],"each":[83],"turn":[85],"along":[86],"four":[87,133],"dimensions":[88,134],"(goal":[89],"alignment,":[90],"efficiency,":[91],"information":[92],"utilization,":[93],"result":[95,138],"verification),":[96],"anchored":[97],"by":[98],"12,541":[99],"task-specific":[100],"rubric":[101],"items.":[102],"Evaluating":[103],"21":[104],"models":[105],"over":[106],"16,000+":[107],"trials,":[108],"find":[110],"that:":[111],"(1)":[112],"effectively":[115],"failure":[119],"dimensions,":[123],"filtering":[124,160],"passes":[126],"invisible":[127],"outcome-only":[129],"evaluation;":[130],"(2)":[131],"the":[132,141,145],"complementary,":[136],"verification":[139],"as":[140],"systematic":[142],"bottleneck;":[143],"(3)":[144],"framework":[146],"is":[147],"robust":[148],"evaluator":[150],"choice":[151],"different":[153],"LLMs;":[155],"(4)":[157],"process-based":[158],"trajectory":[159],"yields":[161],"consistent":[162],"post-training":[163],"improvements":[164],"model":[166],"scales.":[167]},"counts_by_year":[],"updated_date":"2026-08-01T09:00:35.917206","created_date":"2026-08-01T00:00:00"}