{"id":"https://openalex.org/W7161033739","doi":"https://doi.org/10.48550/arxiv.2605.12483","title":"Beyond GRPO and On-Policy Distillation: An Empirical Sparse-to-Dense Reward Principle for Language-Model Post-Training","display_name":"Beyond GRPO and On-Policy Distillation: An Empirical Sparse-to-Dense Reward Principle for Language-Model Post-Training","publication_year":2026,"publication_date":"2026-05-12","ids":{"openalex":"https://openalex.org/W7161033739","doi":"https://doi.org/10.48550/arxiv.2605.12483"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.12483","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12483","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.12483","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5126942858","display_name":"Yuanda Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Yuanda","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027724386","display_name":"Hejian Sang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sang, Hejian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136030250","display_name":"Zhengze Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Zhengze","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136061558","display_name":"Ran He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Ran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136049839","display_name":"Zhipeng Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zhipeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5059864597","display_name":"Alborz Geramifard","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Geramifard, Alborz","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.15850000083446503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.15850000083446503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.13689999282360077,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.07069999724626541,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/verifiable-secret-sharing","display_name":"Verifiable secret sharing","score":0.7290999889373779},{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.6317999958992004},{"id":"https://openalex.org/keywords/transfer","display_name":"Transfer (computing)","score":0.4997999966144562},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.49050000309944153},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.42969998717308044},{"id":"https://openalex.org/keywords/downstream","display_name":"Downstream (manufacturing)","score":0.41370001435279846},{"id":"https://openalex.org/keywords/paraphrase","display_name":"Paraphrase","score":0.4050999879837036},{"id":"https://openalex.org/keywords/upstream","display_name":"Upstream (networking)","score":0.36970001459121704}],"concepts":[{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.7290999889373779},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6884999871253967},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.6317999958992004},{"id":"https://openalex.org/C2776175482","wikidata":"https://www.wikidata.org/wiki/Q1195816","display_name":"Transfer (computing)","level":2,"score":0.4997999966144562},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.49050000309944153},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.42969998717308044},{"id":"https://openalex.org/C2776207758","wikidata":"https://www.wikidata.org/wiki/Q5303302","display_name":"Downstream (manufacturing)","level":2,"score":0.41370001435279846},{"id":"https://openalex.org/C2780922921","wikidata":"https://www.wikidata.org/wiki/Q255189","display_name":"Paraphrase","level":2,"score":0.4050999879837036},{"id":"https://openalex.org/C191172861","wikidata":"https://www.wikidata.org/wiki/Q7899321","display_name":"Upstream (networking)","level":2,"score":0.36970001459121704},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.3614000082015991},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.3571000099182129},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.3555999994277954},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3393999934196472},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.33009999990463257},{"id":"https://openalex.org/C180518391","wikidata":"https://www.wikidata.org/wiki/Q355217","display_name":"Time allocation","level":2,"score":0.32010000944137573},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.29589998722076416},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2906999886035919},{"id":"https://openalex.org/C2992734406","wikidata":"https://www.wikidata.org/wiki/Q413267","display_name":"One shot","level":2,"score":0.2897000014781952},{"id":"https://openalex.org/C48677424","wikidata":"https://www.wikidata.org/wiki/Q6888088","display_name":"Mode (computer interface)","level":2,"score":0.2840999960899353},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2838999927043915},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2703000009059906},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.26919999718666077},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.26010000705718994}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.12483","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12483","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.12483","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12483","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.732793927192688,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"In":[0],"settings":[1],"where":[2,23],"labeled":[3,78,225],"verifiable":[4,116],"training":[5],"data":[6,79,226],"is":[7,25,40,57,171,220],"the":[8,18,82,88,134,141,158,174,186,228,249],"binding":[9],"constraint,":[10],"each":[11,169],"checked":[12],"example":[13],"should":[14],"be":[15],"allocated":[16],"to":[17,221],"model":[19],"and":[20,48,120,191,243],"reward":[21,39,234,246],"density":[22],"it":[24],"most":[26,41],"informative.":[27],"We":[28,95],"identify":[29],"a":[30,65,72,100,178,214],"reward-density":[31],"principle":[32,70],"that":[33,45,62,168],"governs":[34],"this":[35,97],"allocation:":[36],"sparse":[37,233,245],"sequence-level":[38],"useful":[42],"on":[43,81,115,140,147,152,211,227],"models":[44],"can":[46],"explore":[47],"discover":[49],"better":[50,58],"behavior,":[51],"while":[52,155],"dense":[53,93,135,238],"token-level":[54],"teacher":[55,104,131,160,176,180],"supervision":[56],"suited":[59],"for":[60,235,240],"compressing":[61],"behavior":[63,90],"into":[64],"smaller":[66],"deployment":[67],"model.":[68],"The":[69,197,217],"yields":[71],"simple":[73],"allocation":[74],"rule:":[75],"use":[76,232],"scarce":[77,224],"upstream":[80],"strongest":[83],"available":[84],"teacher,":[85],"then":[86],"transfer":[87,156,202,208,239],"reward-shaped":[89],"downstream":[91],"as":[92],"supervision.":[94],"evaluate":[96],"rule":[98],"through":[99,133],"four-stage":[101],"workflow":[102],"--":[103,114,200,209],"RL,":[105],"forward-KL":[106,187],"warmup,":[107],"on-policy":[108,193],"distillation,":[109],"optional":[110],"post-bridge":[111],"student":[112,143,241],"RL":[113,162],"math":[117],"with":[118,177,213],"Qwen3":[119],"Llama":[121],"models.":[122],"At":[123],"fixed":[124],"Qwen3-1.7B":[125],"deployment-student":[126],"size,":[127],"an":[128],"RL-improved":[129,175],"8B":[130],"distilled":[132],"bridge":[136],"outperforms":[137],"direct":[138,204],"GRPO":[139,205],"same":[142,159],"($79.3\\%$":[144],"vs.\\":[145,150],"$75.9\\%$":[146],"MATH;":[148],"$25.2\\%$":[149],"$19.8\\%$":[151],"AIME~2024,":[153],"avg@16),":[154],"from":[157],"\\emph{before}":[161],"underperforms.":[163],"A":[164],"component":[165],"ablation":[166],"confirms":[167],"stage":[170],"load-bearing:":[172],"replacing":[173],"raw":[179],"costs":[181,189,195],"$7.8$":[182],"MATH":[183],"points,":[184],"removing":[185,192],"warmup":[188],"$1.7$,":[190],"distillation":[194],"$3.3$.":[196],"teacher-quality":[198],"ordering":[199],"raw-teacher":[201],"$<$":[203,206],"RL-teacher":[207],"replicates":[210],"Llama-3.1-8B-Instruct":[212],"Llama-3.3-70B-Instruct":[215],"teacher.":[216],"operational":[218],"lesson":[219],"avoid":[222],"spending":[223],"least":[229],"prepared":[230],"policy:":[231],"teacher-side":[236],"discovery,":[237],"compression,":[242],"student-side":[244],"only":[247],"after":[248],"bridge.":[250]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-14T00:00:00"}