{"id":"https://openalex.org/W7140523709","doi":"https://doi.org/10.48550/arxiv.2603.23871","title":"HDPO: Hybrid Distillation Policy Optimization via Privileged Self-Distillation","display_name":"HDPO: Hybrid Distillation Policy Optimization via Privileged Self-Distillation","publication_year":2026,"publication_date":"2026-03-25","ids":{"openalex":"https://openalex.org/W7140523709","doi":"https://doi.org/10.48550/arxiv.2603.23871"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2603.23871","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.23871","pdf_url":"https://arxiv.org/pdf/2603.23871","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2603.23871","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5130715535","display_name":"Ken Ding","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Ding, Ken","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5130715535"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8217999935150146,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8217999935150146,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.032999999821186066,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.03290000185370445,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.5799999833106995},{"id":"https://openalex.org/keywords/mathematical-reasoning","display_name":"mathematical reasoning","score":0.8999999761581421},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"reinforcement learning","score":0.5979999899864197},{"id":"https://openalex.org/keywords/exploration-exploitation-trade-off","display_name":"exploration-exploitation balance","score":0.9049999713897705}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6735000014305115},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.640500009059906},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.4948999881744385},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.448199987411499},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.4189999997615814},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.4153999984264374},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4097000062465668},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.3946000039577484},{"id":"https://openalex.org/C2776378722","wikidata":"https://www.wikidata.org/wiki/Q3454417","display_name":"Realizability","level":2,"score":0.3707999885082245},{"id":"https://openalex.org/C91575142","wikidata":"https://www.wikidata.org/wiki/Q1971426","display_name":"Optimal control","level":2,"score":0.33570000529289246},{"id":"https://openalex.org/C50897621","wikidata":"https://www.wikidata.org/wiki/Q2665508","display_name":"Hybrid system","level":2,"score":0.32659998536109924},{"id":"https://openalex.org/C75553542","wikidata":"https://www.wikidata.org/wiki/Q178161","display_name":"A priori and a posteriori","level":2,"score":0.2987000048160553},{"id":"https://openalex.org/C48677424","wikidata":"https://www.wikidata.org/wiki/Q6888088","display_name":"Mode (computer interface)","level":2,"score":0.28949999809265137},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.28600001335144043},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.2849000096321106},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2840999960899353},{"id":"https://openalex.org/C110121322","wikidata":"https://www.wikidata.org/wiki/Q865811","display_name":"Distribution (mathematics)","level":2,"score":0.2770000100135803}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2603.23871","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.23871","pdf_url":"https://arxiv.org/pdf/2603.23871","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2603.23871","doi":"https://doi.org/10.48550/arxiv.2603.23871","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.23871","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2603.23871","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.23871","pdf_url":"https://arxiv.org/pdf/2603.23871","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.5}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7140523709.pdf","grobid_xml":"https://content.openalex.org/works/W7140523709.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7171978335","https://openalex.org/W7127640081","https://openalex.org/W7160611348","https://openalex.org/W7164364162","https://openalex.org/W7125861034","https://openalex.org/W7215106332","https://openalex.org/W7160905086","https://openalex.org/W7155410664","https://openalex.org/W7160918678","https://openalex.org/W7127356272"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"trained":[3],"with":[4,52,76,136,155],"reinforcement":[5],"learning":[6,34],"(RL)":[7],"for":[8,80],"mathematical":[9],"reasoning":[10],"face":[11],"a":[12],"fundamental":[13],"challenge:":[14],"on":[15,134],"problems":[16],"the":[17,27,74,85,90,97,107,124,130,156,164],"model":[18,75],"cannot":[19],"solve":[20],"at":[21],"all":[22,66],"-":[23,26,100,106],"\"cliff\"":[24],"prompts":[25,64],"RL":[28,51,127],"gradient":[29],"vanishes":[30],"entirely,":[31],"preventing":[32],"any":[33],"signal":[35],"from":[36],"reaching":[37],"these":[38],"failure":[39],"modes.":[40],"We":[41,116],"introduce":[42],"Hybrid":[43],"Distillation":[44],"Policy":[45],"Optimization":[46],"(HDPO),":[47],"which":[48],"augments":[49],"standard":[50],"privileged":[53,70,121],"self-distillation":[54],"targeting":[55],"cliff":[56],"prompts.":[57],"On":[58],"each":[59],"training":[60],"step,":[61],"HDPO":[62,140],"identifies":[63],"where":[65],"rollouts":[67,71],"fail,":[68],"generates":[69],"by":[72,146,149],"providing":[73,160],"ground-truth":[77],"information,":[78],"filters":[79],"correct":[81],"solutions,":[82],"and":[83,94],"distills":[84],"teacher's":[86],"token-level":[87],"distribution":[88],"into":[89],"student.":[91],"Because":[92],"teacher":[93],"student":[95],"share":[96],"same":[98],"weights":[99],"differing":[101],"only":[102],"in":[103,129],"their":[104],"input":[105],"realizability":[108],"gap":[109],"is":[110],"provably":[111],"bounded,":[112],"unlike":[113],"cross-model":[114],"distillation.":[115],"prove":[117],"that":[118,139],"R=1":[119],"filtered":[120],"generation":[122],"recovers":[123],"optimal":[125],"KL-regularized":[126],"policy":[128],"hard-threshold":[131],"limit.":[132],"Experiments":[133],"OpenMathInstruct-2":[135],"Qwen2.5-Math-1.5B-Instruct":[137],"show":[138],"consistently":[141],"improves":[142],"coverage":[143],"metrics":[144],"(pass@4":[145],"+0.8-1.1%,":[147],"pass@8":[148],"+0.4-1.7%)":[150],"while":[151],"maintaining":[152],"greedy":[153],"accuracy,":[154],"distillation":[157],"weight":[158],"lambda":[159],"direct":[161],"control":[162],"over":[163],"exploration-exploitation":[165],"tradeoff.":[166]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-03-27T00:00:00"}