{"id":"https://openalex.org/W7158535599","doi":"https://doi.org/10.48550/arxiv.2604.26326","title":"Addressing Performance Saturation for LLM RL via Precise Entropy Curve Control","display_name":"Addressing Performance Saturation for LLM RL via Precise Entropy Curve Control","publication_year":2026,"publication_date":"2026-04-29","ids":{"openalex":"https://openalex.org/W7158535599","doi":"https://doi.org/10.48550/arxiv.2604.26326"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.26326","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.26326","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.26326","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134882240","display_name":"Bolian Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Bolian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134879247","display_name":"Yifan Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yifan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134908055","display_name":"Yi Ding","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, Yi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120283113","display_name":"Anamika Lochab","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lochab, Anamika","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134878753","display_name":"Ananth Grama","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Grama, Ananth","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5134918312","display_name":"Ruqi Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Ruqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.23890000581741333,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.23890000581741333,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.11739999800920486,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.09470000118017197,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.6345999836921692},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5478000044822693},{"id":"https://openalex.org/keywords/regularization","display_name":"Regularization (linguistics)","score":0.4560999870300293},{"id":"https://openalex.org/keywords/principle-of-maximum-entropy","display_name":"Principle of maximum entropy","score":0.36239999532699585},{"id":"https://openalex.org/keywords/control-theory","display_name":"Control theory (sociology)","score":0.33329999446868896},{"id":"https://openalex.org/keywords/schedule","display_name":"Schedule","score":0.275299996137619}],"concepts":[{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.6345999836921692},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5478000044822693},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5432999730110168},{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.4560999870300293},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3691999912261963},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.367000013589859},{"id":"https://openalex.org/C9679016","wikidata":"https://www.wikidata.org/wiki/Q1417473","display_name":"Principle of maximum entropy","level":2,"score":0.36239999532699585},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.3560999929904938},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.33480000495910645},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.33329999446868896},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.28279998898506165},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.275299996137619},{"id":"https://openalex.org/C125252325","wikidata":"https://www.wikidata.org/wiki/Q1345213","display_name":"Entropy rate","level":4,"score":0.2732999920845032},{"id":"https://openalex.org/C124551494","wikidata":"https://www.wikidata.org/wiki/Q3055345","display_name":"Differential entropy","level":4,"score":0.26089999079704285},{"id":"https://openalex.org/C207821765","wikidata":"https://www.wikidata.org/wiki/Q405372","display_name":"Instability","level":2,"score":0.25540000200271606},{"id":"https://openalex.org/C91575142","wikidata":"https://www.wikidata.org/wiki/Q1971426","display_name":"Optimal control","level":2,"score":0.2515000104904175},{"id":"https://openalex.org/C60507348","wikidata":"https://www.wikidata.org/wiki/Q6795892","display_name":"Maximum entropy probability distribution","level":3,"score":0.2502000033855438}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.26326","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.26326","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.26326","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.26326","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1],"(RL)":[2],"has":[3],"enabled":[4],"complex":[5],"reasoning":[6],"abilities":[7],"in":[8,43,64],"large":[9],"language":[10],"models":[11],"(LLMs).":[12],"However,":[13,56],"most":[14],"RL":[15,25,119],"algorithms":[16],"suffer":[17],"from":[18],"performance":[19,70,152],"saturation,":[20,153],"preventing":[21,49],"continued":[22],"gains":[23],"as":[24],"training":[26],"scales.":[27],"This":[28,113],"problem":[29],"can":[30],"be":[31],"characterized":[32],"by":[33,87,184],"the":[34,65,89,107,115,187],"collapse":[35,51],"of":[36,117,129],"entropy,":[37],"a":[38,78,126,143,164],"key":[39],"diagnostic":[40],"for":[41,174],"exploration":[42],"RL.":[44],"Existing":[45],"attempts":[46],"focus":[47],"on":[48],"entropy":[50,59,85,104,130],"through":[52],"regularization":[53,96],"or":[54],"clipping.":[55],"their":[57],"resulting":[58],"curves":[60],"often":[61],"exhibit":[62],"instability":[63],"long":[66],"term,":[67],"which":[68,132,137],"hinders":[69],"gains.":[71],"In":[72],"this":[73],"paper,":[74],"we":[75,101],"introduce":[76],"Entrocraft,":[77],"simple":[79],"rejection-sampling":[80],"approach":[81],"that":[82,134],"realizes":[83],"user-customized":[84],"schedule":[86],"biasing":[88],"advantage":[90,108],"distributions.":[91],"Entrocraft":[92,123,150],"requires":[93],"no":[94],"objective":[95],"and":[97,120,140,159,181],"is":[98],"advantage-estimator-agnostic.":[99],"Theoretically,":[100],"relate":[102],"per-step":[103],"change":[105],"to":[106,142,167,176],"distribution":[109],"under":[110],"minimal":[111],"assumptions.":[112],"explains":[114],"behavior":[116],"existing":[118],"entropy-preserving":[121],"methods.":[122],"also":[124],"enables":[125,163],"systematic":[127],"study":[128],"schedules,":[131],"reveals":[133],"linear":[135],"annealing,":[136],"starts":[138],"high":[139],"decays":[141],"slightly":[144],"lower":[145],"target,":[146],"performs":[147],"best.":[148],"Empirically,":[149],"addresses":[151],"significantly":[154],"improving":[155],"generalization,":[156],"output":[157],"diversity,":[158],"long-term":[160],"training.":[161],"It":[162],"4B":[165],"model":[166],"outperform":[168],"an":[169],"8B":[170],"baseline,":[171],"sustains":[172],"improvement":[173],"up":[175],"4x":[177],"longer":[178],"before":[179],"plateauing,":[180],"raises":[182],"pass@K":[183],"50%":[185],"over":[186],"baseline.":[188]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-01T00:00:00"}