{"id":"https://openalex.org/W7149873390","doi":"https://doi.org/10.18653/2026.realm-1.28","title":"ToolMisuseBench: An Offline Deterministic Benchmark for Tool Misuse and Recovery in Agentic Systems","display_name":"ToolMisuseBench: An Offline Deterministic Benchmark for Tool Misuse and Recovery in Agentic Systems","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7149873390","doi":"https://doi.org/10.18653/2026.realm-1.28"},"language":"en","primary_location":{"id":"doi:10.18653/2026.realm-1.28","doi":"https://doi.org/10.18653/2026.realm-1.28","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.realm-1.28","pdf_url":"https://aclanthology.org/2026.realm-1.28.pdf","source":{"id":"https://openalex.org/S7407115554","display_name":"Workshop for Research on Agent Language Models (REALM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://aclanthology.org/2026.realm-1.28.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120707166","display_name":"Rista Baral","orcid":"https://orcid.org/0009-0009-8777-1716","observed_orcids":["https://orcid.org/0009-0009-8777-1716"]},"institutions":[{"id":"https://openalex.org/I86501945","display_name":"University of Delaware","ror":"https://ror.org/01sbq1a82","country_code":"US","type":"education","lineage":["https://openalex.org/I86501945"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rista Baral","raw_affiliation_strings":["Akshey Sigdel Independent Researcher , USA","University of Delaware , Newark , DE , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Akshey Sigdel Independent Researcher , USA","institution_ids":[]},{"raw_affiliation_string":"University of Delaware , Newark , DE , USA","institution_ids":["https://openalex.org/I86501945"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5119589722","display_name":"Akshey Sigdel","orcid":"https://orcid.org/0009-0003-2525-8518","observed_orcids":["https://orcid.org/0009-0003-2525-8518"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Akshey Sigdel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":8.2459,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.97386282,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"435","last_page":"442"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.40869998931884766,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.40869998931884766,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.14810000360012054,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.06689999997615814,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/tool-misuse","display_name":"tool misuse","score":0.9039999842643738},{"id":"https://openalex.org/keywords/agentic-systems","display_name":"agentic systems","score":0.8970000147819519},{"id":"https://openalex.org/keywords/rate-limits","display_name":"rate limits","score":0.4189999997615814}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7404000163078308},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7293000221252441},{"id":"https://openalex.org/C52146309","wikidata":"https://www.wikidata.org/wiki/Q7431116","display_name":"Schema (genetic algorithms)","level":2,"score":0.46880000829696655},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.4361000061035156},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.40619999170303345},{"id":"https://openalex.org/C108759981","wikidata":"https://www.wikidata.org/wiki/Q788590","display_name":"Authorization","level":2,"score":0.38830000162124634},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.30820000171661377},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.3043000102043152},{"id":"https://openalex.org/C2775928411","wikidata":"https://www.wikidata.org/wiki/Q2041312","display_name":"Fault injection","level":3,"score":0.25839999318122864},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.2524000108242035},{"id":"https://openalex.org/C2780102126","wikidata":"https://www.wikidata.org/wiki/Q10928179","display_name":"Online and offline","level":2,"score":0.2515000104904175},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2508000135421753}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.18653/2026.realm-1.28","doi":"https://doi.org/10.18653/2026.realm-1.28","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.realm-1.28","pdf_url":"https://aclanthology.org/2026.realm-1.28.pdf","source":{"id":"https://openalex.org/S7407115554","display_name":"Workshop for Research on Agent Language Models (REALM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2604.01508","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2604.01508","pdf_url":"https://arxiv.org/pdf/2604.01508","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.18653/2026.realm-1.28","doi":"https://doi.org/10.18653/2026.realm-1.28","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.realm-1.28","pdf_url":"https://aclanthology.org/2026.realm-1.28.pdf","source":{"id":"https://openalex.org/S7407115554","display_name":"Workshop for Research on Agent Language Models (REALM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7149873390.pdf","grobid_xml":"https://content.openalex.org/works/W7149873390.grobid-xml"},"referenced_works_count":13,"referenced_works":["https://openalex.org/W4385327683","https://openalex.org/W4385474529","https://openalex.org/W4385682136","https://openalex.org/W4387164935","https://openalex.org/W4388964563","https://openalex.org/W4389518608","https://openalex.org/W4399836817","https://openalex.org/W4402387511","https://openalex.org/W7133215859","https://openalex.org/W7138121684","https://openalex.org/W7139921447","https://openalex.org/W7163681859","https://openalex.org/W7165870665"],"related_works":["https://openalex.org/W7165870665","https://openalex.org/W7220737992","https://openalex.org/W7167627487","https://openalex.org/W7160993917","https://openalex.org/W7167621924","https://openalex.org/W7220930860","https://openalex.org/W7163681859","https://openalex.org/W7202365551","https://openalex.org/W7214902728","https://openalex.org/W7221828739"],"abstract_inverted_index":{"Reliable":[0],"tool":[1,17,28,42],"use":[2],"is":[3,136],"a":[4,138,171],"fundamental":[5],"capability":[6],"for":[7,40,176],"agentic":[8],"systems,":[9],"yet":[10],"existing":[11],"evaluations":[12],"primarily":[13],"focus":[14],"on":[15],"successful":[16],"execution":[18,193],"and":[19,44,54,61,72,81,96,114,124,146,154,173],"provide":[20],"limited":[21],"insight":[22],"into":[23],"how":[24],"agents":[25,105,191],"recover":[26],"from":[27],"failures.To":[29],"address":[30],"this":[31],"gap,":[32],"we":[33,101],"introduce":[34],"ToolMisuseBench,":[35],"an":[36],"offline":[37],"deterministic":[38,103],"benchmark":[39],"evaluating":[41],"misuse":[43],"fault-conditioned":[45],"recovery.ToolMisuseBench":[46],"evaluates":[47],"recovery":[48,127,135,160,179],"behavior":[49],"across":[50],"domains,":[51],"fault":[52,182],"types,":[53],"difficulty":[55],"levels,":[56],"using":[57],"output-grounded":[58],"success":[59,123,133,149,165],"criteria":[60],"typed":[62],"recoverability":[63],"metadata.The":[64],"seed-42":[65],"release":[66],"includes":[67],"6,000":[68],"training,":[69],"900":[70],"development,":[71],"1,200":[73],"public-test":[74,117],"tasks":[75],"covering":[76],"CRUD,":[77],"retrieval,":[78],"file":[79],"operations,":[80],"scheduling,":[82],"including":[83],"failures":[84],"such":[85],"as":[86],"timeouts,":[87],"rate":[88],"limits,":[89],"hidden":[90],"first-call":[91],"schema":[92],"perturbations,":[93],"authorization":[94],"errors,":[95],"adversarial":[97],"responses.Using":[98],"the":[99,116],"benchmark,":[100],"evaluate":[102],"reference":[104],"together":[106],"with":[107,130],"local":[108],"3-4B":[109],"tool-calling":[110],"LLMs":[111],"(Qwen3-4B,":[112],"Ministral-3B,":[113,145],"Llama-3.2-3B).On":[115],"split,":[118],"Parser+Repair":[119],"achieves":[120],"0.767":[121],"task":[122,132,148,164],"0.736":[125],"conditional":[126],"success,":[128],"compared":[129],"0.158":[131],"when":[134],"disabled.On":[137],"shared":[139],"stratified":[140],"216-task":[141],"diagnostic":[142],"panel,":[143],"Qwen3-4B,":[144],"Llama-3.2-3Bachieve":[147],"rates":[150],"of":[151,188],"0.796,":[152],"0.903,":[153],"0.773,":[155],"respectively.Fault-level":[156],"analysis":[157],"reveals":[158],"distinct":[159],"weaknesses":[161],"that":[162],"overall":[163],"alone":[166],"fails":[167],"to":[168],"capture.ToolMisuseBench":[169],"provides":[170],"controlled":[172],"reproducible":[174],"framework":[175],"diagnosing":[177],"tooluse":[178],"under":[180],"frozen":[181],"plans,":[183],"enabling":[184],"more":[185],"rigorous":[186],"evaluation":[187],"robust":[189],"tool-using":[190],"beyond":[192],"accuracy.":[194]},"counts_by_year":[{"year":2026,"cited_by_count":6}],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-04-05T00:00:00"}