{"id":"https://openalex.org/W4407764166","doi":"https://doi.org/10.48550/arxiv.2502.13389","title":"Reasoning with Reinforced Functional Token Tuning","display_name":"Reasoning with Reinforced Functional Token Tuning","publication_year":2025,"publication_date":"2025-02-19","ids":{"openalex":"https://openalex.org/W4407764166","doi":"https://doi.org/10.48550/arxiv.2502.13389"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.13389","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.13389","pdf_url":"https://arxiv.org/pdf/2502.13389","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.13389","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5115603666","display_name":"K. Zhang","orcid":"https://orcid.org/0000-0002-7138-9477","observed_orcids":["https://orcid.org/0000-0002-7138-9477"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Kongcheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yao, Qi","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Qi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057196915","display_name":"Baisheng Lai","orcid":"https://orcid.org/0000-0001-8939-041X","observed_orcids":["https://orcid.org/0000-0001-8939-041X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lai, Baisheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108055047","display_name":"Jiaxing Huang","orcid":"https://orcid.org/0000-0002-5353-7843","observed_orcids":["https://orcid.org/0000-0002-5353-7843"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Jiaxing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111361888","display_name":"Wenkai Fang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fang, Wenkai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074103823","display_name":"Dacheng Tao","orcid":"https://orcid.org/0000-0001-7225-5449","observed_orcids":["https://orcid.org/0000-0001-7225-5449","https://orcid.org/0000-0002-5979-578X","https://orcid.org/0000-0002-9209-036X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tao, Dacheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026532752","display_name":"Mingli Song","orcid":"https://orcid.org/0000-0003-2621-6048","observed_orcids":["https://orcid.org/0000-0003-2621-6048","https://orcid.org/0009-0007-5092-0365"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Mingli","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5090854835","display_name":"Shunyu Liu","orcid":"https://orcid.org/0000-0003-0584-9129","observed_orcids":["https://orcid.org/0000-0003-0584-9129","https://orcid.org/0009-0009-5289-1400"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Shunyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":17,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7949000000953674,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7949000000953674,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.08699999749660492,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.024000000208616257,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7889999747276306},{"id":"https://openalex.org/keywords/reinforcement-learning-from-human-feedback","display_name":"reinforcement learning from human feedback","score":0.7590000033378601},{"id":"https://openalex.org/keywords/chain-of-thought","display_name":"chain-of-thought","score":0.7440000176429749},{"id":"https://openalex.org/keywords/online-reinforcement-learning","display_name":"online reinforcement learning","score":0.7319999933242798},{"id":"https://openalex.org/keywords/mathematical-reasoning","display_name":"mathematical reasoning","score":0.8140000104904175},{"id":"https://openalex.org/keywords/math-benchmark","display_name":"MATH benchmark","score":0.5220000147819519}],"concepts":[{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.8233257532119751},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5094476342201233},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.17612555623054504}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.13389","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.13389","pdf_url":"https://arxiv.org/pdf/2502.13389","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.13389","doi":"https://doi.org/10.48550/arxiv.2502.13389","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.13389","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.13389","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.13389","pdf_url":"https://arxiv.org/pdf/2502.13389","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G8235466701","display_name":null,"funder_award_id":"2022R52046","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8310007292","display_name":"\u591a\u5143\u7a7a\u95f4\u878d\u5408\u7684\u8d85\u5927\u89c4\u6a21\u65f6\u5e8f\u56fe\u667a\u80fd\u8ba1\u7b97\u7406\u8bba\u4e0e\u65b9\u6cd5\u7814\u7a76","funder_award_id":"LHZSD24F020001","funder_id":"https://openalex.org/F4320338464","funder_display_name":"Natural Science Foundation of Zhejiang Province"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320338464","display_name":"Natural Science Foundation of Zhejiang Province","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4407764166.pdf","grobid_xml":"https://content.openalex.org/works/W4407764166.grobid-xml"},"referenced_works_count":29,"referenced_works":["https://openalex.org/W3177813494","https://openalex.org/W3195140297","https://openalex.org/W3210904070","https://openalex.org/W4327810158","https://openalex.org/W4385970164","https://openalex.org/W4391631327","https://openalex.org/W4396650433","https://openalex.org/W4399696137","https://openalex.org/W4400023054","https://openalex.org/W4400141325","https://openalex.org/W4400337884","https://openalex.org/W4401306886","https://openalex.org/W4402387391","https://openalex.org/W4402667093","https://openalex.org/W4402733601","https://openalex.org/W4403746689","https://openalex.org/W4404987723","https://openalex.org/W4404987838","https://openalex.org/W4405354744","https://openalex.org/W4405355299","https://openalex.org/W4405655184","https://openalex.org/W4405766390","https://openalex.org/W4405903187","https://openalex.org/W4406231568","https://openalex.org/W4406774573","https://openalex.org/W4414281281","https://openalex.org/W4415070954","https://openalex.org/W6884962196","https://openalex.org/W6959934489"],"related_works":["https://openalex.org/W4391013232","https://openalex.org/W4404782353","https://openalex.org/W4417100863","https://openalex.org/W4416237822","https://openalex.org/W4389820663","https://openalex.org/W4404792954","https://openalex.org/W4407186893","https://openalex.org/W4386554674","https://openalex.org/W4414684647","https://openalex.org/W4417170767"],"abstract_inverted_index":{"In":[0],"this":[1],"work,":[2],"we":[3],"propose":[4],"Reinforced":[5],"Functional":[6],"Token":[7],"Tuning":[8],"(RFTT),":[9],"a":[10,32],"novel":[11],"reinforced":[12],"fine-tuning":[13,63],"framework":[14],"that":[15],"empowers":[16],"Large":[17],"Language":[18],"Models":[19],"(LLMs)":[20],"with":[21,51,74,151],"self-play":[22],"learn-to-reason":[23],"capabilities.":[24],"Unlike":[25],"prior":[26],"prompt-driven":[27,65],"reasoning":[28,54,100],"efforts,":[29],"RFTT":[30,57,125,148],"embeds":[31],"rich":[33],"set":[34],"of":[35,122,147],"learnable":[36],"functional":[37,75,103,115],"tokens":[38,85],"(e.g.,":[39],",":[40,41],")":[42],"directly":[43],"into":[44],"the":[45,80,95,120,123,141,145],"model":[46,81,96],"vocabulary,":[47],"enabling":[48],"chain-of-thought":[49],"construction":[50],"diverse":[52],"human-like":[53],"behaviors.":[55],"Specifically,":[56],"comprises":[58],"two":[59],"phases:":[60],"(1)":[61],"supervised":[62],"performs":[64],"tree":[66],"search":[67,153],"to":[68,82,97,133,138],"obtain":[69],"self-generated":[70],"training":[71],"data":[72],"annotated":[73],"tokens,":[76],"which":[77],"warms":[78],"up":[79],"learn":[83],"these":[84],"for":[86,114],"reasoning;":[87],"and":[88,135],"(2)":[89],"online":[90],"reinforcement":[91],"learning":[92],"further":[93],"allows":[94],"explore":[98],"different":[99],"pathways":[101],"through":[102],"token":[104],"sampling":[105],"without":[106],"relying":[107],"on":[108,126,140],"prompts,":[109],"thereby":[110],"facilitating":[111],"effective":[112],"self-improvement":[113],"reasoning.":[116],"Extensive":[117],"experiments":[118],"demonstrate":[119],"superiority":[121],"proposed":[124],"mathematical":[127],"benchmarks,":[128],"significantly":[129],"boosting":[130],"Qwen-2.5-7B-Instruct":[131],"(70.6%":[132],"79.8%)":[134],"LLaMA-3.1-8B-Instruct":[136],"(32.2%":[137],"60.2%)":[139],"MATH":[142],"dataset.":[143],"Moreover,":[144],"performance":[146],"consistently":[149],"improves":[150],"more":[152],"rollouts":[154],"at":[155,162],"inference":[156],"time.":[157],"Our":[158],"code":[159],"is":[160],"available":[161],"https://github.com/sastpg/RFTT.":[163]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":11}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-02-21T00:00:00"}