{"id":"https://openalex.org/W7140276309","doi":"https://doi.org/10.48550/arxiv.2603.22446","title":"Sparse but Critical: A Token-Level Analysis of Distributional Shifts in RLVR Fine-Tuning of LLMs","display_name":"Sparse but Critical: A Token-Level Analysis of Distributional Shifts in RLVR Fine-Tuning of LLMs","publication_year":2026,"publication_date":"2026-03-23","ids":{"openalex":"https://openalex.org/W7140276309","doi":"https://doi.org/10.48550/arxiv.2603.22446"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.22446","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.22446","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.22446","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5130567742","display_name":"Haoming Meng","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Meng, Haoming","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130609113","display_name":"Kexin Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Kexin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130612174","display_name":"Shaohang Wei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Shaohang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100934186","display_name":"Chiyu Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Chiyu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130596973","display_name":"Shuo Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Shuo","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130629597","display_name":"Xue Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xue","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130620832","display_name":"Guoyin Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Guoyin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130603522","display_name":"Bolin Ding","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, Bolin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5130594641","display_name":"Jingren Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Jingren","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5130567742"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2493000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2493000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11642","display_name":"Genomics and Rare Diseases","score":0.11540000140666962,"subfield":{"id":"https://openalex.org/subfields/1311","display_name":"Genetics"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.09229999780654907,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.732200026512146},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5767999887466431},{"id":"https://openalex.org/keywords/base","display_name":"Base (topology)","score":0.5612999796867371},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.48190000653266907},{"id":"https://openalex.org/keywords/swap","display_name":"Swap (finance)","score":0.46709999442100525},{"id":"https://openalex.org/keywords/contrast","display_name":"Contrast (vision)","score":0.4584999978542328},{"id":"https://openalex.org/keywords/fraction","display_name":"Fraction (chemistry)","score":0.3840999901294708}],"concepts":[{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.732200026512146},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5767999887466431},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.5612999796867371},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5519999861717224},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.48190000653266907},{"id":"https://openalex.org/C99821215","wikidata":"https://www.wikidata.org/wiki/Q1136583","display_name":"Swap (finance)","level":2,"score":0.46709999442100525},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.4584999978542328},{"id":"https://openalex.org/C149629883","wikidata":"https://www.wikidata.org/wiki/Q660926","display_name":"Fraction (chemistry)","level":2,"score":0.3840999901294708},{"id":"https://openalex.org/C207390915","wikidata":"https://www.wikidata.org/wiki/Q1230525","display_name":"Divergence (linguistics)","level":2,"score":0.3797999918460846},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.36649999022483826},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.3571000099182129},{"id":"https://openalex.org/C155076860","wikidata":"https://www.wikidata.org/wiki/Q1932004","display_name":"Token economy","level":3,"score":0.3546999990940094},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33959999680519104},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.2948000133037567},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.29190000891685486},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2786000072956085},{"id":"https://openalex.org/C43091099","wikidata":"https://www.wikidata.org/wiki/Q1067788","display_name":"Through-the-lens metering","level":3,"score":0.26669999957084656},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.25360000133514404},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.25040000677108765}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.22446","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.22446","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.22446","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.22446","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1],"with":[2,86,149],"verifiable":[3],"rewards":[4],"(RLVR)":[5],"has":[6],"significantly":[7],"improved":[8],"reasoning":[9,59],"in":[10],"large":[11],"language":[12],"models":[13,148],"(LLMs),":[14],"yet":[15],"the":[16,51,72,98,106,127,144,210,231],"token-level":[17,40,54,196,241],"mechanisms":[18],"underlying":[19],"these":[20,69,111,131],"improvements":[21,222],"remain":[22],"unclear.":[23],"We":[24,75,103,153],"present":[25],"a":[26,88,158,174,192,214,239,248],"systematic":[27],"empirical":[28],"study":[29],"of":[30,42,53,68,91,110,115,122,130,161,178,195,209],"RLVR's":[31,201],"distributional":[32,43,55,232],"effects":[33],"organized":[34],"around":[35],"three":[36],"main":[37],"analyses:":[38],"(1)":[39],"characterization":[41],"shifts":[44,56,70,112],"between":[45,97,143],"base":[46,99,145,165,179,189],"and":[47,64,83,100,108,120,146,237],"RL":[48,78,101,147,169],"models,":[49],"(2)":[50],"impact":[52],"on":[57,230],"sequence-level":[58],"performance":[60,170,187,202],"through":[61,113],"cross-sampling":[62,136],"interventions,":[63],"(3)":[65],"fine-grained":[66],"mechanics":[67],"at":[71],"token":[73,92,116,141,180],"level.":[74],"find":[76],"that":[77,138,155,218],"fine-tuning":[79,246],"induces":[80],"highly":[81],"sparse":[82,132],"targeted":[84,249],"changes,":[85,133],"only":[87,157],"small":[89,159,176,193],"fraction":[90,160],"distributions":[93],"exhibiting":[94],"meaningful":[95],"divergence":[96],"policies.":[102],"further":[104],"characterize":[105],"structure":[107],"evolution":[109],"analyses":[114],"entropy,":[117],"positional":[118],"concentration,":[119],"reallocation":[121],"probability":[123],"mass.":[124],"To":[125],"assess":[126],"functional":[128],"importance":[129],"we":[134,205],"conduct":[135],"experiments":[137],"selectively":[139],"swap":[140],"choices":[142,181],"varying":[150],"intervention":[151],"budgets.":[152],"show":[154],"inserting":[156],"RL-sampled":[162],"tokens":[163],"into":[164,182],"generations":[166],"progressively":[167],"recovers":[168],"gains,":[171],"while":[172],"injecting":[173],"similarly":[175],"number":[177],"otherwise":[183],"RL-generated":[184],"sequences":[185],"collapses":[186],"to":[188],"levels,":[190],"isolating":[191],"set":[194],"decisions":[197],"directly":[198],"responsible":[199],"for":[200,243],"gains.":[203],"Finally,":[204],"explore":[206],"divergence-weighted":[207],"variants":[208],"advantage":[211],"signal":[212],"as":[213,247],"diagnostic":[215],"intervention,":[216],"finding":[217],"they":[219],"can":[220],"yield":[221],"over":[223],"baselines.":[224],"Together,":[225],"our":[226],"results":[227],"shed":[228],"light":[229],"changes":[233],"induced":[234],"by":[235],"RLVR":[236,245],"provide":[238],"fine-grained,":[240],"lens":[242],"understanding":[244],"refinement":[250],"process.":[251]},"counts_by_year":[],"updated_date":"2026-03-26T06:10:45.909354","created_date":"2026-03-26T00:00:00"}