{"id":"https://openalex.org/W7162778809","doi":"https://doi.org/10.48550/arxiv.2605.29089","title":"OISD: On-Policy Internal Self-Distillation of Language Models","display_name":"OISD: On-Policy Internal Self-Distillation of Language Models","publication_year":2026,"publication_date":"2026-05-27","ids":{"openalex":"https://openalex.org/W7162778809","doi":"https://doi.org/10.48550/arxiv.2605.29089"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.29089","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.29089","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.29089","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137354891","display_name":"Xinyu Liu","orcid":"https://orcid.org/0009-0003-1690-4186"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xinyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137346830","display_name":"Darryl Cherian Jacob","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jacob, Darryl Cherian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137346762","display_name":"Yang Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Yang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137400506","display_name":"Jindong Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jindong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137355237","display_name":"Pan He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Pan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.2921999990940094,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.2921999990940094,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.15549999475479126,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.08160000294446945,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5371000170707703},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.5277000069618225},{"id":"https://openalex.org/keywords/layer","display_name":"Layer (electronics)","score":0.5080000162124634},{"id":"https://openalex.org/keywords/policy-learning","display_name":"Policy learning","score":0.44780001044273376},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.41019999980926514},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.3334999978542328}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7078999876976013},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5388000011444092},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5371000170707703},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5277000069618225},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.5080000162124634},{"id":"https://openalex.org/C2779436431","wikidata":"https://www.wikidata.org/wiki/Q30672407","display_name":"Policy learning","level":2,"score":0.44780001044273376},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.41019999980926514},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3499999940395355},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3422999978065491},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3334999978542328},{"id":"https://openalex.org/C140331021","wikidata":"https://www.wikidata.org/wiki/Q1868104","display_name":"Logit","level":2,"score":0.3230000138282776},{"id":"https://openalex.org/C2781311116","wikidata":"https://www.wikidata.org/wiki/Q83306","display_name":"Group (periodic table)","level":2,"score":0.3118000030517578},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.27059999108314514},{"id":"https://openalex.org/C28427503","wikidata":"https://www.wikidata.org/wiki/Q13580300","display_name":"Internal model","level":3,"score":0.2587999999523163}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.29089","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.29089","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.29089","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.29089","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.40185055136680603,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"reinforcement":[1],"learning":[2],"(RL)":[3],"post-training":[4],"approaches":[5],"primarily":[6],"optimize":[7],"the":[8,39,51,66,72,116,120,157],"final":[9,52,67,117],"output":[10],"policy":[11,73,147],"using":[12],"sparse":[13],"outcome-level":[14],"rewards,":[15],"while":[16,145],"largely":[17],"overlooking":[18],"predictive":[19,48],"signals":[20,49],"encoded":[21],"in":[22],"intermediate":[23,55,81,122,143],"representations.":[24,56],"In":[25],"this":[26],"paper,":[27],"we":[28],"introduce":[29],"a":[30,75,150],"new":[31],"paradigm":[32],"called":[33],"on-policy":[34,47],"internal":[35,77],"self-distillation":[36],"and":[37,59,74,104,163],"propose":[38],"OISD":[40],"framework,":[41],"which":[42,83,96,107],"improves":[43],"reasoning":[44,99,168,174],"by":[45],"transferring":[46],"from":[50,115],"layer":[53,68,118],"to":[54,86,102,113,119,140],"During":[57],"rollout":[58],"Group":[60],"Relative":[61],"Policy":[62],"Optimization":[63],"(GRPO)":[64],"optimization,":[65],"acts":[69],"as":[70],"both":[71,124],"detached":[76],"teacher":[78],"for":[79],"selected":[80,121],"layers,":[82],"are":[84],"guided":[85],"align":[87],"with":[88,133,161],"it":[89],"through":[90],"two":[91],"complementary":[92],"mechanisms:":[93],"logit":[94],"alignment,":[95,106],"transfers":[97],"high-level":[98],"behaviors":[100],"(how":[101],"think),":[103],"attention":[105,110],"enforces":[108],"consistent":[109,164],"patterns":[111],"(where":[112],"look)":[114],"layer,":[123],"without":[125],"requiring":[126],"external":[127],"privileged":[128],"information.":[129],"Our":[130],"OISD,":[131,160],"together":[132],"GRPO,":[134],"employs":[135],"signed":[136],"advantage-weighted":[137],"Jensen--Shannon":[138],"alignment":[139],"distill":[141],"informative":[142],"representations":[144],"preserving":[146],"consistency":[148],"under":[149],"unified":[151],"acting":[152],"policy.":[153],"Experimental":[154],"results":[155],"demonstrate":[156],"effectiveness":[158],"of":[159],"substantial":[162],"improvements":[165],"over":[166],"strong":[167],"RL":[169],"baselines":[170],"across":[171],"four":[172],"mathematical":[173],"tasks.":[175],"The":[176],"code":[177],"will":[178],"be":[179],"released":[180],"at":[181],"https://github.com/THE-MALT-LAB/OISD":[182]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-30T00:00:00"}