{"id":"https://openalex.org/W4415316687","doi":"https://doi.org/10.48550/arxiv.2510.06664","title":"ToolMem: Enhancing Multimodal Agents with Learnable Tool Capability Memory","display_name":"ToolMem: Enhancing Multimodal Agents with Learnable Tool Capability Memory","publication_year":2025,"publication_date":"2025-10-08","ids":{"openalex":"https://openalex.org/W4415316687","doi":"https://doi.org/10.48550/arxiv.2510.06664"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2510.06664","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.06664","pdf_url":"https://arxiv.org/pdf/2510.06664","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2510.06664","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Xiao, Yunzhong","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiao, Yunzhong","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027712402","display_name":"Yangmin Li","orcid":"https://orcid.org/0000-0002-4448-3310","observed_orcids":["https://orcid.org/0000-0002-4448-3310"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li, Yangmin","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101836291","display_name":"Hewei Wang","orcid":"https://orcid.org/0000-0002-6952-0886","observed_orcids":["https://orcid.org/0000-0002-6952-0886"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Hewei","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020961043","display_name":"Yunlong Tang","orcid":"https://orcid.org/0000-0002-9711-3380","observed_orcids":["https://orcid.org/0000-0002-9711-3380"]},"institutions":[{"id":"https://openalex.org/I5388228","display_name":"University of Rochester","ror":"https://ror.org/022kthw22","country_code":"US","type":"education","lineage":["https://openalex.org/I5388228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tang, Yunlong","raw_affiliation_strings":["University of Rochester"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Rochester","institution_ids":["https://openalex.org/I5388228"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5041297063","display_name":"Zora Zhiruo Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Zora Zhiruo","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":17,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9711999893188477,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9711999893188477,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.013000000268220901,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.0035000001080334187,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/tool-selection","display_name":"tool selection","score":0.7509999871253967},{"id":"https://openalex.org/keywords/multimodal-agents","display_name":"multimodal agents","score":0.7929999828338623},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.6679999828338623},{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.9829999804496765},{"id":"https://openalex.org/keywords/text-generation","display_name":"text generation","score":0.7599999904632568},{"id":"https://openalex.org/keywords/text-to-image-generation","display_name":"text-to-image generation","score":0.9919999837875366}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7953000068664551},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.6931999921798706},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6473000049591064},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6114000082015991},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5591999888420105},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.546999990940094},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.49639999866485596},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.4115999937057495},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3862000107765198},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.35420000553131104},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.3208000063896179},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3107999861240387},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.28690001368522644},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2590999901294708}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2510.06664","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.06664","pdf_url":"https://arxiv.org/pdf/2510.06664","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2510.06664","doi":"https://doi.org/10.48550/arxiv.2510.06664","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.06664","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2510.06664","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.06664","pdf_url":"https://arxiv.org/pdf/2510.06664","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415316687.pdf","grobid_xml":"https://content.openalex.org/works/W4415316687.grobid-xml"},"referenced_works_count":29,"referenced_works":["https://openalex.org/W2167839676","https://openalex.org/W2609826708","https://openalex.org/W2962985038","https://openalex.org/W3007672467","https://openalex.org/W3015883388","https://openalex.org/W3027879771","https://openalex.org/W3135367836","https://openalex.org/W4226278401","https://openalex.org/W4323717348","https://openalex.org/W4378505261","https://openalex.org/W4380136738","https://openalex.org/W4380993798","https://openalex.org/W4385474529","https://openalex.org/W4386065691","https://openalex.org/W4390872747","https://openalex.org/W4391212748","https://openalex.org/W4393147158","https://openalex.org/W4399911164","https://openalex.org/W4403622204","https://openalex.org/W4405714829","https://openalex.org/W4407415573","https://openalex.org/W4407759090","https://openalex.org/W4411119262","https://openalex.org/W4415800703","https://openalex.org/W4416729942","https://openalex.org/W4417239966","https://openalex.org/W7131067892","https://openalex.org/W7133214447","https://openalex.org/W7133236450"],"related_works":["https://openalex.org/W7164043410","https://openalex.org/W7140221938","https://openalex.org/W7168259939","https://openalex.org/W6891839109","https://openalex.org/W4409264021","https://openalex.org/W4411700982","https://openalex.org/W7125351917","https://openalex.org/W7222595481","https://openalex.org/W7163148686","https://openalex.org/W7163220138"],"abstract_inverted_index":{"Agents":[0],"utilizing":[1],"tools":[2,27,36,44,84],"powered":[3],"by":[4,85,127,204],"large":[5],"language":[6],"models":[7,11],"(LLMs)":[8],"or":[9],"vision-language":[10],"(VLMs)":[12],"have":[13],"demonstrated":[14],"remarkable":[15],"progress":[16],"in":[17,50,64,136,210],"diverse":[18],"tasks":[19,156],"across":[20,39,189],"text":[21,165,190],"and":[22,89,131,133,148,167,185,191,206],"visual":[23],"modalities.":[24],"Unlike":[25],"traditional":[26],"such":[28],"as":[29],"calculators,":[30],"which":[31],"give":[32],"deterministic":[33],"outputs,":[34],"neural":[35,170],"perform":[37],"uncertainly":[38],"task":[40,47],"scenarios.":[41,194,212],"While":[42],"different":[43,83],"for":[45,70],"a":[46,100],"may":[48],"excel":[49],"varied":[51,164],"scenarios,":[52],"existing":[53],"agents":[54,105,117,180],"typically":[55],"rely":[56],"on":[57,162],"fixed":[58],"tools,":[59],"thus":[60],"limiting":[61],"the":[62,66,80,95,140,150],"flexibility":[63],"selecting":[65],"most":[67],"suitable":[68],"tool":[69,97,122,152,182,199],"specific":[71],"tasks.":[72],"In":[73],"contrast,":[74],"humans":[75],"snowball":[76],"their":[77,129],"understanding":[78],"of":[79,82,121],"capabilities":[81,123],"interacting":[86],"with":[87],"them,":[88],"apply":[90],"this":[91,110],"knowledge":[92],"to":[93,118,153,173],"select":[94,149],"optimal":[96,198],"when":[98],"solving":[99],"future":[101],"task.":[102],"To":[103],"build":[104],"that":[106,115],"similarly":[107],"benefit":[108],"from":[109,124,146],"process,":[111],"we":[112,177],"propose":[113],"ToolMem":[114,161,196],"enables":[116],"develop":[119],"memories":[120],"previous":[125],"interactions,":[126],"summarizing":[128],"strengths":[130],"weaknesses":[132],"storing":[134],"them":[135],"memory;":[137],"at":[138],"inference,":[139],"agent":[141],"can":[142],"retrieve":[143],"relevant":[144],"entries":[145],"ToolMem,":[147],"best":[151],"solve":[154],"individual":[155],"more":[157,187],"accurately.":[158],"We":[159],"evaluate":[160],"learning":[163],"generation":[166,169,193],"text-to-image":[168],"tools.":[171],"Compared":[172],"no-memory,":[174],"generic":[175],"agents,":[176],"find":[178],"ToolMem-augmented":[179],"predict":[181],"performance":[183],"14.8%":[184],"28.7%":[186],"accurately":[188],"multimodal":[192],"Moreover,":[195],"facilitates":[197],"selection":[200],"among":[201],"multiple":[202],"choices":[203],"21%":[205],"24%":[207],"absolute":[208],"increases":[209],"respective":[211]},"counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":4}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-18T00:00:00"}