{"id":"https://openalex.org/W7138873852","doi":"https://doi.org/10.48550/arxiv.2603.16157","title":"DyJR: Preserving Diversity in Reinforcement Learning with Verifiable Rewards via Dynamic Jensen-Shannon Replay","display_name":"DyJR: Preserving Diversity in Reinforcement Learning with Verifiable Rewards via Dynamic Jensen-Shannon Replay","publication_year":2026,"publication_date":"2026-03-17","ids":{"openalex":"https://openalex.org/W7138873852","doi":"https://doi.org/10.48550/arxiv.2603.16157"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.16157","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.16157","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.16157","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5130026494","display_name":"Long Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Li, Long","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129894591","display_name":"Zhijian Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Zhijian","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130148857","display_name":"Tianyi Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Tianyi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101160945","display_name":"Weidi Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Weidi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111194074","display_name":"Zuming Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Zuming","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130002960","display_name":"Wei Chu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chu, Wei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129876217","display_name":"Zhe Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zhe","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129881361","display_name":"Shirui Pan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pan, Shirui","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129797022","display_name":"Chao Qu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qu, Chao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5130052328","display_name":"Yuan Qi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi, Yuan","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5130026494"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2094999998807907,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2094999998807907,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.20190000534057617,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.07490000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7321000099182129},{"id":"https://openalex.org/keywords/synchronizing","display_name":"Synchronizing","score":0.5641999840736389},{"id":"https://openalex.org/keywords/verifiable-secret-sharing","display_name":"Verifiable secret sharing","score":0.44530001282691956},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.3869999945163727},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.3605000078678131},{"id":"https://openalex.org/keywords/constraint","display_name":"Constraint (computer-aided design)","score":0.36039999127388},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.3070000112056732},{"id":"https://openalex.org/keywords/flexibility","display_name":"Flexibility (engineering)","score":0.30559998750686646}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7943000197410583},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7321000099182129},{"id":"https://openalex.org/C162932704","wikidata":"https://www.wikidata.org/wiki/Q1058791","display_name":"Synchronizing","level":3,"score":0.5641999840736389},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.44530001282691956},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4262000024318695},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3874000012874603},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.3869999945163727},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.3605000078678131},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.36039999127388},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.34389999508857727},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.3070000112056732},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.30559998750686646},{"id":"https://openalex.org/C37404715","wikidata":"https://www.wikidata.org/wiki/Q380679","display_name":"Dynamic programming","level":2,"score":0.289900004863739},{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.28859999775886536},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.28279998898506165},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2752000093460083},{"id":"https://openalex.org/C48677424","wikidata":"https://www.wikidata.org/wiki/Q6888088","display_name":"Mode (computer interface)","level":2,"score":0.27309998869895935},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.2696000039577484},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.26829999685287476},{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.25929999351501465},{"id":"https://openalex.org/C125112378","wikidata":"https://www.wikidata.org/wiki/Q176640","display_name":"Randomness","level":2,"score":0.25839999318122864},{"id":"https://openalex.org/C2777767291","wikidata":"https://www.wikidata.org/wiki/Q1080291","display_name":"Sizing","level":2,"score":0.2567000091075897},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2531000077724457},{"id":"https://openalex.org/C2777632111","wikidata":"https://www.wikidata.org/wiki/Q1937518","display_name":"Reservation","level":2,"score":0.2515999972820282}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.16157","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.16157","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.16157","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.16157","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"While":[0],"Reinforcement":[1],"Learning":[2],"(RL)":[3],"enhances":[4,171],"Large":[5],"Language":[6],"Model":[7],"reasoning,":[8],"on-policy":[9],"algorithms":[10],"like":[11],"GRPO":[12,139],"are":[13],"sample-inefficient":[14],"as":[15,140,142,145],"they":[16],"discard":[17],"past":[18],"rollouts.":[19],"Existing":[20],"experience":[21],"replay":[22],"methods":[23],"address":[24],"this":[25,35,62],"by":[26],"reusing":[27],"accurate":[28],"samples":[29],"for":[30],"direct":[31,116],"policy":[32],"updates,":[33],"but":[34],"often":[36],"incurs":[37],"high":[38],"computational":[39],"costs":[40],"and":[41,96,109,131,147,173],"causes":[42],"mode":[43],"collapse":[44],"via":[45],"overfitting.":[46],"We":[47],"argue":[48],"that":[49,93,135,169],"historical":[50],"data":[51],"should":[52],"prioritize":[53],"sustaining":[54],"diversity":[55,125,172],"rather":[56],"than":[57],"simply":[58],"reinforcing":[59],"accuracy.":[60],"To":[61],"end,":[63],"we":[64,167],"propose":[65],"Dynamic":[66,91],"Jensen-Shannon":[67,111],"Replay":[68],"(DyJR),":[69],"a":[70,77,120],"simple":[71],"yet":[72],"effective":[73],"regularization":[74],"framework":[75],"using":[76],"dynamic":[78],"reference":[79],"distribution":[80],"from":[81,159],"recent":[82],"trajectories.":[83],"DyJR":[84,136,170,184],"introduces":[85],"two":[86],"innovations:":[87],"(1)":[88],"A":[89],"Time-Sensitive":[90],"Buffer":[92],"uses":[94],"FIFO":[95],"adaptive":[97],"sizing":[98],"to":[99,123,154],"retain":[100],"only":[101],"temporally":[102],"proximal":[103],"samples,":[104],"synchronizing":[105],"with":[106,119],"model":[107],"evolution;":[108],"(2)":[110],"Divergence":[112],"Regularization,":[113],"which":[114],"replaces":[115],"gradient":[117],"updates":[118],"distributional":[121],"constraint":[122],"prevent":[124],"collapse.":[126],"Experiments":[127],"on":[128,176],"mathematical":[129],"reasoning":[130],"Text-to-SQL":[132],"benchmarks":[133],"demonstrate":[134],"significantly":[137],"outperforms":[138],"well":[141],"baselines":[143],"such":[144],"RLEP":[146],"Ex-GRPO,":[148],"while":[149],"maintaining":[150],"training":[151,187],"efficiency":[152],"comparable":[153],"the":[155,160,186],"original":[156],"GRPO.":[157],"Furthermore,":[158],"perspective":[161],"of":[162,183],"Rank-$k$":[163],"token":[164],"probability":[165],"evolution,":[166],"show":[168],"mitigates":[174],"over-reliance":[175],"Rank-1":[177],"tokens,":[178],"elucidating":[179],"how":[180],"specific":[181],"sub-modules":[182],"influence":[185],"dynamics.":[188]},"counts_by_year":[],"updated_date":"2026-03-20T20:54:20.808490","created_date":"2026-03-20T00:00:00"}