{"id":"https://openalex.org/W7160632323","doi":"https://doi.org/10.48550/arxiv.2605.06523","title":"On the Implicit Reward Overfitting and the Low-rank Dynamics in RLVR","display_name":"On the Implicit Reward Overfitting and the Low-rank Dynamics in RLVR","publication_year":2026,"publication_date":"2026-05-07","ids":{"openalex":"https://openalex.org/W7160632323","doi":"https://doi.org/10.48550/arxiv.2605.06523"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.06523","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.06523","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.06523","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135690303","display_name":"Hao Ye","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ye, Hao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058458910","display_name":"Jisheng Dang","orcid":"https://orcid.org/0000-0002-5378-6225"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dang, Jisheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135650095","display_name":"Junfeng Fang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fang, Junfeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135673548","display_name":"Bimei Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Bimei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135674755","display_name":"Yizhou Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yizhou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101846529","display_name":"Ning Lv","orcid":"https://orcid.org/0000-0003-2770-8714"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lv, Ning","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067111051","display_name":"Wencan Zhang","orcid":"https://orcid.org/0000-0001-8657-2883"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Wencan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135712580","display_name":"Hong Peng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Hong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135704875","display_name":"Bin Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Bin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135656778","display_name":"Tat-Seng Chua","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chua, Tat-Seng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.4747999906539917,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.4747999906539917,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.12600000202655792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.04899999871850014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/overfitting","display_name":"Overfitting","score":0.9140999913215637},{"id":"https://openalex.org/keywords/counterintuitive","display_name":"Counterintuitive","score":0.7226999998092651},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6146000027656555},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.585099995136261},{"id":"https://openalex.org/keywords/verifiable-secret-sharing","display_name":"Verifiable secret sharing","score":0.4799000024795532},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.46810001134872437},{"id":"https://openalex.org/keywords/dynamics","display_name":"Dynamics (music)","score":0.4081000089645386}],"concepts":[{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.9140999913215637},{"id":"https://openalex.org/C101097943","wikidata":"https://www.wikidata.org/wiki/Q5176983","display_name":"Counterintuitive","level":2,"score":0.7226999998092651},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6169000267982483},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6146000027656555},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.585099995136261},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5558000206947327},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.4799000024795532},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.46810001134872437},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.45680001378059387},{"id":"https://openalex.org/C145912823","wikidata":"https://www.wikidata.org/wiki/Q113558","display_name":"Dynamics (music)","level":2,"score":0.4081000089645386},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3937000036239624},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3580999970436096},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3181999921798706},{"id":"https://openalex.org/C169903167","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Test set","level":2,"score":0.2718000113964081},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.26930001378059387},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.26809999346733093},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.25060001015663147}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.06523","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.06523","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.06523","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.06523","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"extensive":[1],"research":[2],"has":[3],"demonstrated":[4],"that":[5,145],"the":[6,24,48,52,59,70,126,143],"enhanced":[7],"reasoning":[8,96],"capabilities":[9],"acquired":[10],"by":[11,102],"models":[12],"through":[13],"Reinforcement":[14],"Learning":[15],"with":[16,131],"Verifiable":[17],"Rewards":[18],"(RLVR)":[19],"are":[20],"primarily":[21],"concentrated":[22],"within":[23],"rank-1":[25,85,132],"components.":[26],"Predicated":[27],"on":[28,58],"this":[29],"observation,":[30],"we":[31,74],"employed":[32],"Periodic":[33],"Rank-1":[34],"Substitution":[35],"and":[36,157,166],"identified":[37],"a":[38,104,135],"counterintuitive":[39],"phenomenon:":[40],"RLVR":[41,88,99,146,162],"may":[42],"exhibit":[43],"implicit":[44],"reward":[45],"overfitting":[46],"to":[47,179],"training":[49,71,177],"dataset.":[50],"Specifically,":[51],"model":[53,92,120,164],"can":[54],"achieve":[55],"satisfactory":[56],"performance":[57],"test":[60],"set":[61],"even":[62],"when":[63],"its":[64],"rewards":[65],"remain":[66],"relatively":[67],"low":[68],"during":[69,139],"process.":[72],"Furthermore,":[73],"characterize":[75],"three":[76],"distinct":[77],"properties":[78],"of":[79,110,113],"RL":[80,173],"training:":[81],"(1)":[82],"The":[83,108],"effective":[84],"component":[86],"in":[87,118,151],"don't":[89],"maintain":[90],"other":[91,176],"knowledge":[93],"except":[94],"mathematical":[95],"capability.":[97],"(2)":[98],"fundamentally":[100],"functions":[101],"optimizing":[103,148],"specific":[105],"singular":[106,111,128],"spectrum.":[107],"distribution":[109],"values":[112],"almost":[114],"all":[115],"linear":[116],"layers":[117],"RLVR-trained":[119],"behaves":[121],"like":[122],"heavy-tailed":[123],"distribution.":[124],"(3)":[125],"left":[127],"vectors":[129],"associated":[130],"components":[133],"demonstrate":[134],"stronger":[136],"alignment":[137],"tendency":[138],"training,":[140],"which":[141],"echoes":[142],"discovery":[144],"is":[147],"sampling":[149],"efficiency":[150],"essence.":[152],"Taken":[153],"together,":[154],"our":[155],"findings":[156],"analysis":[158],"further":[159],"reveal":[160],"how":[161],"shapes":[163],"parameters":[165],"offer":[167],"potential":[168],"insights":[169],"for":[170],"improving":[171],"existing":[172],"paradigms":[174,178],"or":[175],"implement":[180],"continual":[181],"learning.":[182]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-09T00:00:00"}