{"id":"https://openalex.org/W4416017530","doi":"https://doi.org/10.1145/3746252.3761391","title":"StepTool: Enhancing Multi-Step Tool Usage in LLMs via Step-Grained Reinforcement Learning","display_name":"StepTool: Enhancing Multi-Step Tool Usage in LLMs via Step-Grained Reinforcement Learning","publication_year":2025,"publication_date":"2025-11-08","ids":{"openalex":"https://openalex.org/W4416017530","doi":"https://doi.org/10.1145/3746252.3761391"},"language":null,"primary_location":{"id":"doi:10.1145/3746252.3761391","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746252.3761391","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3746252.3761391","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049723987","display_name":"Yuanqing Yu","orcid":"https://orcid.org/0000-0002-2942-1576"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuanqing Yu","raw_affiliation_strings":["DCST, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-2942-1576","affiliations":[{"raw_affiliation_string":"DCST, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078413396","display_name":"Zhefan Wang","orcid":"https://orcid.org/0000-0001-7176-8234"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhefan Wang","raw_affiliation_strings":["DCST, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-7176-8234","affiliations":[{"raw_affiliation_string":"DCST, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101524043","display_name":"Weizhi Ma","orcid":"https://orcid.org/0000-0001-5604-7527"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weizhi Ma","raw_affiliation_strings":["Institute for AI Industry Research, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-5604-7527","affiliations":[{"raw_affiliation_string":"Institute for AI Industry Research, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Shuai Wang","orcid":"https://orcid.org/0009-0002-6297-9085"},"institutions":[{"id":"https://openalex.org/I4210159102","display_name":"Huawei Technologies (Sweden)","ror":"https://ror.org/0500fyd17","country_code":"SE","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210159102"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Shuai Wang","raw_affiliation_strings":["Huawei Noah's Ark Lab, Huawei Noah's Ark Lab, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0002-6297-9085","affiliations":[{"raw_affiliation_string":"Huawei Noah's Ark Lab, Huawei Noah's Ark Lab, Beijing, China","institution_ids":["https://openalex.org/I4210159102"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001967239","display_name":"Chuhan Wu","orcid":"https://orcid.org/0000-0001-5730-8792"},"institutions":[{"id":"https://openalex.org/I4210159102","display_name":"Huawei Technologies (Sweden)","ror":"https://ror.org/0500fyd17","country_code":"SE","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210159102"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Chuhan Wu","raw_affiliation_strings":["Huawei Noah's Ark Lab, Huawei Noah's Ark Lab, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-5730-8792","affiliations":[{"raw_affiliation_string":"Huawei Noah's Ark Lab, Huawei Noah's Ark Lab, Beijing, China","institution_ids":["https://openalex.org/I4210159102"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007911614","display_name":"Zhiqiang Guo","orcid":"https://orcid.org/0000-0001-9393-4854"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiqiang Guo","raw_affiliation_strings":["DCST, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-9393-4854","affiliations":[{"raw_affiliation_string":"DCST, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100402996","display_name":"Min Zhang","orcid":"https://orcid.org/0000-0003-3158-1920"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Min Zhang","raw_affiliation_strings":["DCST, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3158-1920","affiliations":[{"raw_affiliation_string":"DCST, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.7588,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.89372144,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"3952","last_page":"3962"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3052000105381012,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3052000105381012,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.27480000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.10090000182390213,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7293000221252441},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6190000176429749},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5726000070571899},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.529699981212616},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4153999984264374},{"id":"https://openalex.org/keywords/supervised-learning","display_name":"Supervised learning","score":0.36649999022483826},{"id":"https://openalex.org/keywords/task-analysis","display_name":"Task analysis","score":0.3359000086784363}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7656000256538391},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7293000221252441},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6190000176429749},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5929999947547913},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5726000070571899},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.570900022983551},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.529699981212616},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4153999984264374},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.36649999022483826},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3359000086784363},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.30730000138282776},{"id":"https://openalex.org/C2779436431","wikidata":"https://www.wikidata.org/wiki/Q30672407","display_name":"Policy learning","level":2,"score":0.3034999966621399},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.3000999987125397},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2896000146865845},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.27410000562667847},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.27399998903274536},{"id":"https://openalex.org/C28006648","wikidata":"https://www.wikidata.org/wiki/Q6934509","display_name":"Multi-task learning","level":3,"score":0.2671000063419342}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3746252.3761391","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746252.3761391","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3746252.3761391","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746252.3761391","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":5,"referenced_works":["https://openalex.org/W4241811150","https://openalex.org/W4399115306","https://openalex.org/W4402671878","https://openalex.org/W4404594861","https://openalex.org/W4404783862"],"related_works":[],"abstract_inverted_index":{"Despite":[0],"their":[1],"powerful":[2],"text":[3,39],"generation":[4,40],"capabilities,":[5],"large":[6],"language":[7],"models":[8,155],"(LLMs)":[9],"still":[10],"struggle":[11],"to":[12,17,92,103,114],"effectively":[13],"utilize":[14],"external":[15],"tools":[16],"solve":[18],"complex":[19],"tasks,":[20],"a":[21,38,60,67,182],"challenge":[22],"known":[23],"as":[24,37,59,181],"tool":[25,35,57,79,94,176,190],"learning.":[26],"Existing":[27],"methods":[28,113],"primarily":[29],"rely":[30],"on":[31,97],"supervised":[32],"fine-tuning,":[33],"treating":[34],"learning":[36,58,71,177],"problem":[41],"while":[42],"overlooking":[43],"the":[44,116,169],"decision-making":[45,62],"complexities":[46],"inherent":[47],"in":[48,77,137,175,192],"multi-step":[49,78,189],"contexts.":[50],"In":[51],"this":[52],"work,":[53],"we":[54],"propose":[55],"modeling":[56,174],"dynamic":[61],"process":[63],"and":[64,101,106,134,143,178,184,195],"introduce":[65],"StepTool,":[66],"novel":[68],"step-grained":[69],"reinforcement":[70],"framework":[72],"that":[73,128,152],"enhances":[74],"LLMs'":[75],"capabilities":[76],"use.":[80],"StepTool":[81,129,153,180],"comprises":[82],"two":[83],"key":[84],"components:":[85],"Step-grained":[86,107],"Reward":[87],"Shaping,":[88],"which":[89,109],"assigns":[90],"rewards":[91],"each":[93],"interaction":[95],"based":[96],"its":[98],"invocation":[99],"success":[100],"contribution":[102],"task":[104,140],"completion;":[105],"Optimization,":[108],"applies":[110],"policy":[111],"gradient":[112],"optimize":[115],"model":[117],"across":[118,124],"multiple":[119],"decision":[120,173],"steps.":[121],"Extensive":[122],"experiments":[123],"diverse":[125],"benchmarks":[126],"show":[127],"consistently":[130],"outperforms":[131],"both":[132],"SFT-based":[133],"RL-based":[135],"baselines":[136],"terms":[138],"of":[139,145,171],"Pass":[141],"Rate":[142],"Recall":[144],"relevant":[146],"tools.":[147],"Furthermore,":[148],"our":[149],"analysis":[150],"suggests":[151],"helps":[154],"discover":[156],"new":[157],"tool-use":[158],"strategies":[159],"rather":[160],"than":[161],"merely":[162],"re-weighting":[163],"prior":[164],"knowledge.":[165],"These":[166],"results":[167],"highlight":[168],"importance":[170],"fine-grained":[172],"establish":[179],"general":[183],"robust":[185],"solution":[186],"for":[187],"enhancing":[188],"use":[191],"LLMs.":[193],"Code":[194],"data":[196],"are":[197],"available":[198],"at":[199],"https://github.com/yuyq18/StepTool.":[200]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-11-08T00:00:00"}