{"id":"https://openalex.org/W4414946087","doi":"https://doi.org/10.48550/arxiv.2507.11371","title":"Step-wise Policy for Rare-tool Knowledge (SPaRK): Offline RL that Drives Diverse Tool Use in LLMs","display_name":"Step-wise Policy for Rare-tool Knowledge (SPaRK): Offline RL that Drives Diverse Tool Use in LLMs","publication_year":2025,"publication_date":"2025-07-15","ids":{"openalex":"https://openalex.org/W4414946087","doi":"https://doi.org/10.48550/arxiv.2507.11371"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2507.11371","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2507.11371","pdf_url":"https://arxiv.org/pdf/2507.11371","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2507.11371","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119897402","display_name":"Gabriel Bo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bo, Gabriel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119897403","display_name":"Koa Chang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chang, Koa","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5018173821","display_name":"Juping Gu","orcid":"https://orcid.org/0000-0003-0078-4602"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gu, Justin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14330","display_name":"Library Science and Information Systems","score":0.7674999833106995,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14330","display_name":"Library Science and Information Systems","score":0.7674999833106995,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13999","display_name":"Digital Rights Management and Security","score":0.7541000247001648,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12478","display_name":"Wikis in Education and Collaboration","score":0.6899999976158142,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.701200008392334},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.5559999942779541},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.5181000232696533},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.43149998784065247},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.3944000005722046},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.3546999990940094},{"id":"https://openalex.org/keywords/diversity","display_name":"Diversity (politics)","score":0.35199999809265137}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.701200008392334},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6485000252723694},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.5559999942779541},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5558000206947327},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5293999910354614},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.5181000232696533},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.43149998784065247},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3944000005722046},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.36559998989105225},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.3546999990940094},{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.35199999809265137},{"id":"https://openalex.org/C2779436431","wikidata":"https://www.wikidata.org/wiki/Q30672407","display_name":"Policy learning","level":2,"score":0.3407999873161316},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.31130000948905945},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.30809998512268066},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.2903999984264374},{"id":"https://openalex.org/C9679016","wikidata":"https://www.wikidata.org/wiki/Q1417473","display_name":"Principle of maximum entropy","level":2,"score":0.289900004863739},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.2896000146865845},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.2867000102996826}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2507.11371","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2507.11371","pdf_url":"https://arxiv.org/pdf/2507.11371","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2507.11371","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2507.11371","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2507.11371","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2507.11371","pdf_url":"https://arxiv.org/pdf/2507.11371","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0],"present":[1],"Step-wise":[2],"Policy":[3],"for":[4,45],"Rare-tool":[5],"Knowledge":[6],"(SPaRK),":[7],"a":[8,38,52,71,76],"novel":[9],"reinforcement":[10,34],"learning":[11],"framework":[12],"that":[13,42,106,132],"teaches":[14],"large":[15],"language":[16],"models":[17],"to":[18,99,124],"explore":[19],"diverse":[20],"tool":[21,49,121,137],"usage":[22],"patterns":[23],"beyond":[24],"conventional":[25],"high-temperature":[26],"sampling.":[27],"Building":[28],"on":[29,59],"recent":[30],"advances":[31],"in":[32,120],"step-wise":[33],"learning,":[35],"we":[36],"introduce":[37],"dual-objective":[39],"reward":[40],"system":[41],"simultaneously":[43],"optimizes":[44],"answer":[46],"quality":[47],"and":[48,127],"diversity,":[50],"training":[51],"Llama-3.1":[53],"8B":[54],"model":[55],"through":[56,135],"offline":[57],"PPO":[58],"synthetically":[60],"generated":[61],"trajectories":[62],"from":[63],"the":[64,90],"MMLU-Pro":[65,113],"dataset.":[66],"Our":[67],"approach":[68],"uniquely":[69],"employs":[70],"rarity-first":[72],"exploitation":[73],"strategy":[74],"where":[75],"GPT-4o":[77],"judge":[78],"scores":[79],"candidate":[80],"actions":[81],"across":[82,111],"eight":[83],"distinct":[84],"tools":[85,98],"plus":[86],"chain-of-thought":[87],"reasoning,":[88],"with":[89],"policy":[91],"favoring":[92],"less-frequently":[93],"used":[94],"but":[95],"still":[96],"viable":[97],"encourage":[100],"systematic":[101],"exploration.":[102],"Empirical":[103],"results":[104],"demonstrate":[105],"SPaRK":[107],"achieves":[108],"competitive":[109],"performance":[110],"14":[112],"categories":[114],"while":[115],"exhibiting":[116],"significantly":[117],"higher":[118],"entropy":[119],"selection":[122],"compared":[123],"both":[125],"baseline":[126],"supervised":[128],"fine-tuning":[129],"approaches,":[130],"suggesting":[131],"algorithmic":[133],"exploration":[134],"explicit":[136],"diversity":[138],"can":[139],"enhance":[140],"reasoning":[141],"capabilities":[142],"without":[143],"sacrificing":[144],"accuracy.":[145]},"counts_by_year":[],"updated_date":"2026-07-03T08:13:44.112507","created_date":"2025-10-08T00:00:00"}