{"id":"https://openalex.org/W7220699741","doi":"https://doi.org/10.48550/arxiv.2610.05048","title":"E$^2$-OPSD: Taming Entropy Overshoot in On-Policy Self-Distillation","display_name":"E$^2$-OPSD: Taming Entropy Overshoot in On-Policy Self-Distillation","publication_year":2026,"publication_date":"2026-10-04","ids":{"openalex":"https://openalex.org/W7220699741","doi":"https://doi.org/10.48550/arxiv.2610.05048"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.05048","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.05048","pdf_url":"https://arxiv.org/pdf/2610.05048","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.05048","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5157712562","display_name":"Yifei Liu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Liu, Yifei","raw_affiliation_strings":["The Chinese University of Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5157712013","display_name":"Minghao Fang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fang, Minghao","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5157710767","display_name":"Xinyu Gu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Gu, Xinyu","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070810427","display_name":"Chenxu Yao","orcid":"https://orcid.org/0009-0009-4837-673X","observed_orcids":["https://orcid.org/0009-0009-4837-673X"]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yao, Chengkai","raw_affiliation_strings":["Shanghai Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiaotong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5157750160","display_name":"Mengdi Liu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Mengdi","raw_affiliation_strings":["Visual Information Processing and Learning , ICT , CAS"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Visual Information Processing and Learning , ICT , CAS","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061355407","display_name":"Tengfei Ma","orcid":"https://orcid.org/0000-0001-5592-9544","observed_orcids":["https://orcid.org/0000-0001-5592-9544"]},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ma, Tengfei","raw_affiliation_strings":["Hunan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hunan University","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5157672248","display_name":"Jiangbin Zheng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I3133055985","display_name":"Westlake University","ror":"https://ror.org/05hfa4n20","country_code":"CN","type":"education","lineage":["https://openalex.org/I3133055985"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zheng, Jiangbin","raw_affiliation_strings":["Westlake University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Westlake University","institution_ids":["https://openalex.org/I3133055985"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5157758830","display_name":"Chang Yu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu, Chang","raw_affiliation_strings":["Nanjing University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanjing University","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5157771886","display_name":"Zhangyang Gao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Gao, Zhangyang","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4391012619"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":9,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8154000043869019,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8154000043869019,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.050700001418590546,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.020800000056624413,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/on-policy-self-distillation","display_name":"on-policy self-distillation","score":0.9980000257492065},{"id":"https://openalex.org/keywords/token-level-supervision","display_name":"token-level supervision","score":0.609000027179718},{"id":"https://openalex.org/keywords/mathematical-reasoning","display_name":"mathematical reasoning","score":0.6779999732971191},{"id":"https://openalex.org/keywords/out-of-domain-generalization","display_name":"out-of-domain generalization","score":0.6639999747276306},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.4729999899864197}],"concepts":[{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.5888000130653381},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4607999920845032},{"id":"https://openalex.org/C171752962","wikidata":"https://www.wikidata.org/wiki/Q255166","display_name":"Kullback\u2013Leibler divergence","level":2,"score":0.43160000443458557},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3603000044822693},{"id":"https://openalex.org/C83725634","wikidata":"https://www.wikidata.org/wiki/Q7268699","display_name":"Qualitative reasoning","level":2,"score":0.34139999747276306},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.33320000767707825},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32690000534057617},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3138999938964844},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3034999966621399},{"id":"https://openalex.org/C97399411","wikidata":"https://www.wikidata.org/wiki/Q825367","display_name":"Coin flipping","level":2,"score":0.2847000062465668},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.26409998536109924},{"id":"https://openalex.org/C9679016","wikidata":"https://www.wikidata.org/wiki/Q1417473","display_name":"Principle of maximum entropy","level":2,"score":0.262800008058548}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.05048","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.05048","pdf_url":"https://arxiv.org/pdf/2610.05048","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.05048","doi":"https://doi.org/10.48550/arxiv.2610.05048","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.05048","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.05048","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.05048","pdf_url":"https://arxiv.org/pdf/2610.05048","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.6164000034332275,"id":"https://metadata.un.org/sdg/4","display_name":"Quality education"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7220699741.pdf","grobid_xml":"https://content.openalex.org/works/W7220699741.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W3215698779","https://openalex.org/W4300978888","https://openalex.org/W4380993498","https://openalex.org/W4387321091","https://openalex.org/W4391631327","https://openalex.org/W4391631876","https://openalex.org/W4403883066","https://openalex.org/W4406960744","https://openalex.org/W4414281281","https://openalex.org/W4414939355","https://openalex.org/W4415566463","https://openalex.org/W4416048648","https://openalex.org/W6948611197","https://openalex.org/W7125861034","https://openalex.org/W7125915072","https://openalex.org/W7126034091","https://openalex.org/W7128814765","https://openalex.org/W7134813685","https://openalex.org/W7140517510","https://openalex.org/W7150828934","https://openalex.org/W7154507714","https://openalex.org/W7161046900","https://openalex.org/W7161147084","https://openalex.org/W7162137784","https://openalex.org/W7164364162","https://openalex.org/W7164513051","https://openalex.org/W7166760620","https://openalex.org/W7167214743","https://openalex.org/W7167266087","https://openalex.org/W7167642481","https://openalex.org/W7171504029","https://openalex.org/W7196953391","https://openalex.org/W7196965682","https://openalex.org/W7202159004"],"related_works":["https://openalex.org/W7214731163","https://openalex.org/W7214772251","https://openalex.org/W7202181542","https://openalex.org/W7160918678","https://openalex.org/W7223337476","https://openalex.org/W7215074916","https://openalex.org/W7214590818","https://openalex.org/W7220652251","https://openalex.org/W7203880136","https://openalex.org/W7204908525"],"abstract_inverted_index":{"On-policy":[0],"self-distillation":[1],"(OPSD)":[2],"provides":[3],"dense":[4],"token-level":[5],"supervision":[6,82],"without":[7,106,133],"a":[8,29,48,124],"second":[9],"model:":[10],"one":[11],"network":[12],"acts":[13],"as":[14,21],"teacher":[15,64],"with":[16,23,123],"the":[17,25,43,94,102,120,135,145,151,178],"reference":[18],"solution":[19],"and":[20,45,137,153,189],"student":[22,38],"only":[24],"problem.":[26],"We":[27,54,110],"identify":[28],"specific":[30],"failure":[31],"mode":[32],"of":[33,60,155,182],"this":[34,73],"recipe.":[35],"During":[36],"training,":[37],"token":[39],"entropy":[40,52,147],"rises":[41],"past":[42],"teacher's":[44],"remains":[46,198],"elevated,":[47],"pattern":[49],"we":[50],"call":[51],"overshoot.":[53],"trace":[55],"it":[56,108],"to":[57,77,85,113,149,165,184],"both":[58,115],"sides":[59],"distillation.":[61],"The":[62],"reference-conditioned":[63],"is":[65],"confident":[66],"along":[67],"its":[68,81],"answer-directed":[69],"reasoning":[70,91,131,162],"path,":[71],"but":[72],"confidence":[74],"transfers":[75],"poorly":[76],"student-generated":[78],"prefixes,":[79],"making":[80],"overly":[83],"tied":[84],"answer-specific":[86],"cues":[87],"rather":[88],"than":[89],"reusable":[90],"patterns;":[92],"meanwhile,":[93],"forward":[95,203],"KL":[96],"used":[97],"by":[98,163],"OPSD":[99],"continually":[100],"diffuses":[101],"student's":[103],"predictive":[104],"distribution":[105],"pulling":[107],"back.":[109],"introduce":[111],"E$^2$-OPSD":[112,159,197],"address":[114],"causes.":[116],"Exemplar-guided":[117],"teaching":[118],"replaces":[119],"current":[121],"answer":[122],"retrieved":[125],"solved":[126],"neighboring":[127],"problem,":[128],"providing":[129],"transferable":[130],"guidance":[132],"revealing":[134],"destination":[136],"better":[138],"matching":[139],"student-reachable":[140],"states.":[141],"Entropy-aware":[142],"distillation":[143],"uses":[144],"student-teacher":[146],"gap":[148],"determine":[150],"direction":[152],"strength":[154],"each":[156],"token's":[157],"correction.":[158],"improves":[160],"math":[161],"up":[164,183],"4.3":[166],"points":[167,186,191],"in":[168,187,192],"mean@16":[169,188],"over":[170,177],"OPSD,":[171],"while":[172],"out-of-domain":[173],"evaluations":[174],"show":[175],"gains":[176],"corresponding":[179],"base":[180],"models":[181],"4.9":[185],"5.5":[190],"pass@8.":[193],"Despite":[194],"these":[195],"gains,":[196],"simple,":[199],"requiring":[200],"no":[201],"additional":[202],"passes":[204],"or":[205],"networks.":[206],"Code":[207],"repo:":[208],"https://github.com/orionfei/E2-OPSD.":[209]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-10-07T00:00:00"}