{"id":"https://openalex.org/W7164308022","doi":"https://doi.org/10.48550/arxiv.2606.12370","title":"Breaking Entropy Bounds: Accelerating RL Training via MTP with Rejection Sampling","display_name":"Breaking Entropy Bounds: Accelerating RL Training via MTP with Rejection Sampling","publication_year":2026,"publication_date":"2026-06-10","ids":{"openalex":"https://openalex.org/W7164308022","doi":"https://doi.org/10.48550/arxiv.2606.12370"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.12370","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12370","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.12370","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138399424","display_name":"Yucheng Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yucheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070156365","display_name":"Huiqiang Jiang","orcid":"https://orcid.org/0000-0002-1327-4882"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Huiqiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138441516","display_name":"Yang Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Yang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138396770","display_name":"Jianxin Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Jianxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138421044","display_name":"Yi Zhang (9093)","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101918928","display_name":"Yizhong Cao","orcid":"https://orcid.org/0000-0003-0995-6845"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Yizhong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138474582","display_name":"Yuhao Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Yuhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138455229","display_name":"Fan Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Fan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004626105","display_name":"Rui Men","orcid":"https://orcid.org/0000-0002-4429-3461"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Men, Rui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138403462","display_name":"Jianwei Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Jianwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138431232","display_name":"An Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, An","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138448531","display_name":"Bowen Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Bowen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138403944","display_name":"Bo Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Bo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138417849","display_name":"Fei Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Fei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138446647","display_name":"Junyang Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Junyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138464646","display_name":"Dayiheng Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Dayiheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138457741","display_name":"Jingren Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Jingren","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.19840000569820404,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.19840000569820404,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.1080000028014183,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.10300000011920929,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.6489999890327454},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.6488000154495239},{"id":"https://openalex.org/keywords/thompson-sampling","display_name":"Thompson sampling","score":0.5716000199317932},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.546999990940094},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.5349000096321106},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.4864000082015991},{"id":"https://openalex.org/keywords/importance-sampling","display_name":"Importance sampling","score":0.48260000348091125},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.4823000133037567}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7526999711990356},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.6489999890327454},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.6488000154495239},{"id":"https://openalex.org/C73602740","wikidata":"https://www.wikidata.org/wiki/Q7795822","display_name":"Thompson sampling","level":3,"score":0.5716000199317932},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.546999990940094},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.5349000096321106},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4864000082015991},{"id":"https://openalex.org/C52740198","wikidata":"https://www.wikidata.org/wiki/Q1539564","display_name":"Importance sampling","level":3,"score":0.48260000348091125},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.4823000133037567},{"id":"https://openalex.org/C167981619","wikidata":"https://www.wikidata.org/wiki/Q1685498","display_name":"Cross entropy","level":3,"score":0.47279998660087585},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4717000126838684},{"id":"https://openalex.org/C34388435","wikidata":"https://www.wikidata.org/wiki/Q2267362","display_name":"Bounded function","level":2,"score":0.4083000123500824},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3885999917984009},{"id":"https://openalex.org/C9679016","wikidata":"https://www.wikidata.org/wiki/Q1417473","display_name":"Principle of maximum entropy","level":2,"score":0.37940001487731934},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.37630000710487366},{"id":"https://openalex.org/C75782508","wikidata":"https://www.wikidata.org/wiki/Q3333633","display_name":"Cross-entropy method","level":4,"score":0.3483000099658966},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3377000093460083},{"id":"https://openalex.org/C160234255","wikidata":"https://www.wikidata.org/wiki/Q812535","display_name":"Bayesian inference","level":3,"score":0.3190000057220459},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.30309998989105225},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.29330000281333923},{"id":"https://openalex.org/C51823790","wikidata":"https://www.wikidata.org/wiki/Q504353","display_name":"Greedy algorithm","level":2,"score":0.29109999537467957},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.2639999985694885},{"id":"https://openalex.org/C60782215","wikidata":"https://www.wikidata.org/wiki/Q3333679","display_name":"Probabilistic method","level":3,"score":0.2572999894618988},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2508000135421753}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.12370","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12370","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.12370","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12370","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1],"(RL)":[2],"has":[3],"become":[4],"a":[5,30,64,102,158,222],"key":[6,19],"component":[7],"in":[8,21,69,112,130,151,261],"modern":[9],"large":[10],"language":[11],"models,":[12],"yet":[13],"the":[14,18,87,95,108,113,125,141,229,233],"rollout":[15],"stage":[16],"remains":[17],"bottleneck":[20],"RL":[22,50,81,114,131,207,263],"training":[23,144,204,213,264],"pipelines.":[24,82],"Although":[25],"Multi-Token":[26],"Prediction":[27],"(MTP)":[28],"offers":[29],"natural":[31],"solution":[32],"to":[33,53,76,133,178,184,257],"accelerate":[34],"rollouts":[35],"through":[36],"speculative":[37],"decoding,":[38],"many":[39],"studies":[40],"have":[41],"observed":[42],"that":[43,86,119,140,163,210,246],"MTP":[44,68,78,88,143,203,212,238],"acceptance":[45,89,169,173,180,224],"rates":[46,181],"degrade":[47],"significantly":[48],"during":[49,206],"training,":[51],"leading":[52],"limited":[54],"speedup":[55,227],"performance.":[56],"To":[57],"address":[58],"this":[59],"bottleneck,":[60],"we":[61,84,117,156,199],"present":[62],"Bebop,":[63],"systematic":[65],"study":[66],"of":[67,97,110,265],"LLM":[70],"post-training,":[71],"and":[72,154,182,195,208,218,226,244,268],"offer":[73],"practical":[74],"recipes":[75],"integrate":[77],"into":[79],"large-scale":[80],"First,":[83],"reveal":[85],"rate":[90,174,225],"is":[91],"fundamentally":[92],"bounded":[93],"by":[94,128],"fluctuation":[96],"model":[98],"entropy,":[99],"which":[100],"demonstrates":[101],"clear":[103],"negative":[104],"linear":[105],"relationship":[106],"with":[107,214],"rise":[109],"entropy":[111,129],"stage.":[115],"Second,":[116],"show":[118,209,252],"probabilistic":[120],"rejection":[121,167,219],"sampling":[122,168,220],"largely":[123],"alleviates":[124],"disturbance":[126],"introduced":[127],"compared":[132],"greedy":[134],"draft":[135],"sampling.":[136],"We":[137,240],"further":[138],"identify":[139],"conventional":[142],"objectives":[145],"(cross-entropy":[146],"or":[147],"KL)":[148],"are":[149],"suboptimal":[150],"such":[152],"settings,":[153],"therefore":[155],"propose":[157],"novel":[159],"end-to-end":[160,259],"TV":[161,216],"loss":[162,217],"directly":[164],"optimizes":[165],"multi-step":[166],"rate,":[170],"yielding":[171],"~10%":[172],"improvements,":[175],"achieving":[176],"up":[177,183,256],"95%":[179],"25%":[185],"extra":[186],"inference":[187],"throughput":[188],"gains":[189],"across":[190],"mathematical":[191],"reasoning,":[192],"code":[193],"generation,":[194],"agentic":[196],"tasks.":[197],"Third,":[198],"test":[200],"various":[201],"online":[202,237],"strategies":[205],"pre-RL":[211],"e2e":[215],"achieves":[221,255],"consistent":[223],"throughout":[228],"entire":[230],"RL,":[231],"eliminating":[232],"need":[234],"for":[235],"costly":[236],"updating.":[239],"provide":[241],"extensive":[242],"experiments":[243],"analysis":[245],"validate":[247],"our":[248,253],"findings.":[249],"Experimental":[250],"results":[251],"method":[254],"1.8x":[258],"acceleration":[260],"async":[262],"Qwen3.5,":[266],"Qwen3.6,":[267],"Qwen3.7":[269],"models.":[270]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-12T00:00:00"}