{"id":"https://openalex.org/W7160891526","doi":"https://doi.org/10.48550/arxiv.2605.08441","title":"DUET: Optimize Token-Budget Allocation for Reinforcement Learning with Verifiable Rewards","display_name":"DUET: Optimize Token-Budget Allocation for Reinforcement Learning with Verifiable Rewards","publication_year":2026,"publication_date":"2026-05-08","ids":{"openalex":"https://openalex.org/W7160891526","doi":"https://doi.org/10.48550/arxiv.2605.08441"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.08441","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08441","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.08441","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135927973","display_name":"Haoyu Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Haoyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135993877","display_name":"Xuandong Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Xuandong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135391877","display_name":"Xuhai \"Orson'' Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Xuhai \"Orson''","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135923428","display_name":"Nori Jacoby","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jacoby, Nori","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.29280000925064087,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.29280000925064087,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.1200999990105629,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.09690000116825104,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.820900022983551},{"id":"https://openalex.org/keywords/verifiable-secret-sharing","display_name":"Verifiable secret sharing","score":0.6972000002861023},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.695900022983551},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.645799994468689},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6269000172615051},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.5202999711036682},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5098999738693237}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8270999789237976},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.820900022983551},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.6972000002861023},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.695900022983551},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.645799994468689},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6269000172615051},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.5202999711036682},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5098999738693237},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39079999923706055},{"id":"https://openalex.org/C8505890","wikidata":"https://www.wikidata.org/wiki/Q605095","display_name":"Budget constraint","level":2,"score":0.34360000491142273},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.32899999618530273},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.29750001430511475},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.29159998893737793},{"id":"https://openalex.org/C2986053828","wikidata":"https://www.wikidata.org/wiki/Q355217","display_name":"Time budget","level":2,"score":0.288100004196167},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.28360000252723694},{"id":"https://openalex.org/C180518391","wikidata":"https://www.wikidata.org/wiki/Q355217","display_name":"Time allocation","level":2,"score":0.27619999647140503},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.275299996137619},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.260699987411499},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.2551000118255615}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.08441","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08441","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.08441","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08441","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/17","score":0.43773216009140015,"display_name":"Partnerships for the goals"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1,273],"with":[2,15,120,160],"verifiable":[3],"rewards":[4],"(RLVR)":[5],"generates":[6],"hundreds":[7],"of":[8,10,57,104,181,209,271],"thousands":[9],"tokens":[11],"per":[12],"training":[13,81],"step,":[14],"rollout":[16,47],"generation":[17],"dominating":[18],"the":[19,139,161,165,182,206,220,225,230,235,269,272],"computational":[20],"cost.":[21],"The":[22],"overall":[23],"token":[24,183],"budget":[25,74,231],"can":[26],"be":[27],"controlled":[28,54],"along":[29],"two":[30],"main":[31],"dimensions:":[32],"(i)":[33],"deciding":[34,43],"which":[35,239],"prompts":[36],"to":[37,107,123,126,149,234],"allocate":[38],"rollouts":[39,111],"to,":[40],"and":[41,79,115,138,154,156,217,224],"(ii)":[42],"how":[44,109],"long":[45],"each":[46,112],"should":[48],"be.":[49],"Prior":[50],"work":[51],"has":[52],"generally":[53],"only":[55,179,261],"one":[56],"these":[58,250],"dimensions":[59],"at":[60,191],"a":[61,71,92,100,116,172],"time.":[62,82],"We":[63,83,204],"show":[64],"that":[65,98,253],"jointly":[66],"tuning":[67],"both":[68,76],"decisions":[69],"under":[70],"shared":[72],"compute":[73,246],"improves":[75],"reasoning":[77],"quality":[78,244,270],"wall-clock":[80,174],"instantiate":[84],"this":[85],"view":[86],"as":[87,229,245],"\\textbf{DU}al-controlled":[88],"tok\\textbf{E}n":[89],"alloca\\textbf{T}ion":[90],"(DUET),":[91],"computationally":[93],"efficient":[94,240],"layer":[95],"over":[96,201],"GRPO":[97,137],"uses":[99],"lightweight":[101],"pre-rollout":[102],"surrogate":[103],"prompt":[105,113],"informativeness":[106],"set":[108,124],"many":[110],"receives,":[114],"marker-gated":[117],"abort":[118],"rule":[119],"importance":[121],"reweighting":[122],"when":[125],"stop":[127],"them.":[128],"On":[129],"Qwen3-1.7B":[130],"trained":[131],"on":[132,158,164,211],"MATH,":[133],"DUET":[134,185,210,223,254],"outperforms":[135,187],"full-budget":[136,202],"other":[140,150,212],"three":[141],"budget-aware":[142,255],"baseline":[143,163,189,227],"methods.":[144],"DUET's":[145],"advantage":[146],"further":[147],"generalizes":[148],"benchmarks":[151],"across":[152],"math":[153],"coding,":[155],"is":[157],"par":[159],"best":[162],"scientific":[166],"Q\\&A":[167],"domain,":[168],"while":[169],"also":[170,266],"achieving":[171,195],"$1.62\\times$":[173],"speedup.":[175],"More":[176,248],"notably,":[177],"using":[178],"50\\%":[180],"budget,":[184,194],"still":[186],"all":[188],"methods":[190,241],"their":[192],"full":[193],"an":[196],"even":[197],"higher":[198],"$2.51\\times$":[199],"speedup":[200],"GRPO.":[203],"verify":[205],"high":[207],"performance":[208],"backbone":[213],"LLMs,":[214],"including":[215],"Qwen3-4B":[216],"Llama-3.2-3B-Instruct.":[218],"Notably,":[219],"gap":[221],"between":[222],"strongest":[226],"\\emph{widens}":[228],"tightens,":[232],"contrary":[233],"usual":[236],"pattern":[237],"in":[238],"trade":[242],"off":[243],"decreases.":[247],"broadly,":[249],"results":[251],"suggest":[252],"control":[256],"strategies":[257],"are":[258],"valuable":[259],"not":[260],"for":[262,267],"accelerating":[263],"training,":[264],"but":[265],"improving":[268],"signal.":[274]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-13T00:00:00"}