{"id":"https://openalex.org/W4402387511","doi":"https://doi.org/10.48550/arxiv.2408.04682","title":"ToolSandbox: A Stateful, Conversational, Interactive Evaluation Benchmark for LLM Tool Use Capabilities","display_name":"ToolSandbox: A Stateful, Conversational, Interactive Evaluation Benchmark for LLM Tool Use Capabilities","publication_year":2024,"publication_date":"2024-08-08","ids":{"openalex":"https://openalex.org/W4402387511","doi":"https://doi.org/10.48550/arxiv.2408.04682"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2408.04682","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.04682","pdf_url":"https://arxiv.org/pdf/2408.04682","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2408.04682","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100933013","display_name":"Jiarui Lu","orcid":"https://orcid.org/0009-0001-4512-1004"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Jiarui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107099562","display_name":"Thomas Holleis","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Holleis, Thomas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101959000","display_name":"Yizhe Zhang","orcid":"https://orcid.org/0000-0002-9599-7995"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yizhe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107099563","display_name":"Bernhard Aumayer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aumayer, Bernhard","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102228395","display_name":"Nan Feng","orcid":"https://orcid.org/0009-0005-3670-9884"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nan, Feng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109779248","display_name":"Felix Bai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bai, Felix","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115594444","display_name":"Shuang Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Shuang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064498067","display_name":"Shen Ma","orcid":"https://orcid.org/0000-0001-8460-874X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Shen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100722833","display_name":"Mengyu Li","orcid":"https://orcid.org/0000-0002-5015-0328"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Mengyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106247062","display_name":"Guoli Yin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Guoli","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5105989076","display_name":"Zirui Wang","orcid":"https://orcid.org/0009-0006-6976-9522"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zirui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5112901893","display_name":"Ruoming Pang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pang, Ruoming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14330","display_name":"Library Science and Information Systems","score":0.895799994468689,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14330","display_name":"Library Science and Information Systems","score":0.895799994468689,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.8072999715805054,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13523","display_name":"Mathematics, Computing, and Information Processing","score":0.8070999979972839,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/stateful-firewall","display_name":"Stateful firewall","score":0.9483460187911987},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8262079954147339},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7274287343025208},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.06816452741622925},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.04309627413749695}],"concepts":[{"id":"https://openalex.org/C22927095","wikidata":"https://www.wikidata.org/wiki/Q1784206","display_name":"Stateful firewall","level":3,"score":0.9483460187911987},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8262079954147339},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7274287343025208},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.06816452741622925},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.04309627413749695},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C158379750","wikidata":"https://www.wikidata.org/wiki/Q214111","display_name":"Network packet","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2408.04682","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.04682","pdf_url":"https://arxiv.org/pdf/2408.04682","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2408.04682","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2408.04682","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2408.04682","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.04682","pdf_url":"https://arxiv.org/pdf/2408.04682","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2583381754","https://openalex.org/W2144255176","https://openalex.org/W2222333653","https://openalex.org/W109977192","https://openalex.org/W1970797462","https://openalex.org/W2566747981","https://openalex.org/W4312120139","https://openalex.org/W3120111116"],"abstract_inverted_index":{"Recent":[0],"large":[1],"language":[2],"models":[3,90],"(LLMs)":[4],"advancements":[5],"sparked":[6],"a":[7,41,61,70,92],"growing":[8],"research":[9],"interest":[10],"in":[11,107],"tool":[12,54],"assisted":[13],"LLMs":[14],"solving":[15],"real-world":[16],"challenges,":[17],"which":[18],"calls":[19],"for":[20,74],"comprehensive":[21],"evaluation":[22,68,72,125],"of":[23],"tool-use":[24,121],"capabilities.":[25,123],"While":[26],"previous":[27],"works":[28],"focused":[29],"on":[30,40],"either":[31],"evaluating":[32],"over":[33,79],"stateless":[34],"web":[35],"services":[36],"(RESTful":[37],"API),":[38],"based":[39],"single":[42],"turn":[43],"user":[44,63],"prompt,":[45],"or":[46],"an":[47,80],"off-policy":[48],"dialog":[49],"trajectory,":[50],"ToolSandbox":[51,108,124],"includes":[52],"stateful":[53],"execution,":[55],"implicit":[56],"state":[57],"dependencies":[58],"between":[59],"tools,":[60],"built-in":[62],"simulator":[64],"supporting":[65],"on-policy":[66],"conversational":[67],"and":[69,76,88,96,103],"dynamic":[71],"strategy":[73],"intermediate":[75],"final":[77],"milestones":[78],"arbitrary":[81],"trajectory.":[82],"We":[83],"show":[84],"that":[85],"open":[86],"source":[87],"proprietary":[89],"have":[91],"significant":[93],"performance":[94],"gap,":[95],"complex":[97],"tasks":[98],"like":[99],"State":[100],"Dependency,":[101],"Canonicalization":[102],"Insufficient":[104],"Information":[105],"defined":[106],"are":[109],"challenging":[110],"even":[111],"the":[112],"most":[113],"capable":[114],"SOTA":[115],"LLMs,":[116],"providing":[117],"brand-new":[118],"insights":[119],"into":[120],"LLM":[122],"framework":[126],"is":[127],"released":[128],"at":[129],"https://github.com/apple/ToolSandbox":[130]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2025-10-10T00:00:00"}