{"id":"https://openalex.org/W4414859134","doi":"https://doi.org/10.18653/v1/2026.acl-long.74","title":"Harnessing Negative Signals: Reinforcement Distillation from Teacher Data for LLM Reasoning","display_name":"Harnessing Negative Signals: Reinforcement Distillation from Teacher Data for LLM Reasoning","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W4414859134","doi":"https://doi.org/10.18653/v1/2026.acl-long.74"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2026.acl-long.74","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.74","pdf_url":"https://aclanthology.org/2026.acl-long.74.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.acl-long.74.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5025137094","display_name":"Shijing Xu","orcid":"https://orcid.org/0000-0002-5542-7779"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shuyao Xu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011847644","display_name":"Peng Cheng","orcid":"https://orcid.org/0000-0002-1994-893X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng Peng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Jiangxuan Long","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiangxuan Long","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103257775","display_name":"Weidi Xu","orcid":"https://orcid.org/0000-0002-7279-9339"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Weidi Xu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056008057","display_name":"Wei Chu","orcid":"https://orcid.org/0000-0002-0276-224X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei Chu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5107118224","display_name":"Yuan Qi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan Qi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":20.1957,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.96453283,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1618","last_page":"1639"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9715999960899353,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9715999960899353,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9297999739646912,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.7971000075340271},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6312999725341797},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.508899986743927},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.3488999903202057},{"id":"https://openalex.org/keywords/case-based-reasoning","display_name":"Case-based reasoning","score":0.3059000074863434},{"id":"https://openalex.org/keywords/term","display_name":"Term (time)","score":0.29249998927116394},{"id":"https://openalex.org/keywords/reinforcement","display_name":"Reinforcement","score":0.28780001401901245}],"concepts":[{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.7971000075340271},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6730999946594238},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6312999725341797},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6219000220298767},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5846999883651733},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.508899986743927},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.3488999903202057},{"id":"https://openalex.org/C20162079","wikidata":"https://www.wikidata.org/wiki/Q1151406","display_name":"Case-based reasoning","level":2,"score":0.3059000074863434},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.29249998927116394},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.28780001401901245},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2847000062465668},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.2833000123500824},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.28040000796318054},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.27950000762939453},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.2754000127315521},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.26589998602867126},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.2603999972343445},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.250900000333786}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2026.acl-long.74","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.74","pdf_url":"https://aclanthology.org/2026.acl-long.74.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2505.24850","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.24850","pdf_url":"https://arxiv.org/pdf/2505.24850","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2505.24850","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2505.24850","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.acl-long.74","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.74","pdf_url":"https://aclanthology.org/2026.acl-long.74.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4414859134.pdf","grobid_xml":"https://content.openalex.org/works/W4414859134.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"advances":[1],"in":[2,49,99,204,210,401,497],"model":[3,135],"distillation":[4,101,363],"show":[5],"that":[6,77,131,273,400],"data":[7,145,169,191],"from":[8,118,286,444,464],"advanced":[9],"reasoning":[10,22,39,47,194,318,323,488],"models":[11,289],"can":[12,33,309],"effectively":[13,42,196,311],"train":[14],"smaller":[15,288],"student":[16],"models.However,":[17],"standard":[18],"practices":[19],"discard":[20],"incorrect":[21,486],"traces-valuable,":[23],"yet":[24],"underutilized":[25],"data.This":[26,140,299],"paper":[27],"addresses":[28],"the":[29,85,106,119,143,162,172,207,295,302,337,372,385,402,419,442,453,475,480,498],"critical":[30,368],"question:":[31],"How":[32,308],"both":[34,71,198,313,465,483],"positive":[35,63,72,199,314,466],"and":[36,73,97,200,283,315,354,414,435,459,467,485,491],"negative":[37,74,151,201,275,316,445,468],"distilled":[38,202,298,317,328,487],"traces":[40,117,276,319],"be":[41],"leveraged":[43],"to":[44,301,320,360,410],"maximize":[45,321],"LLM":[46,322],"performance":[48,129,176,209,324,389,454],"an":[50,124,156],"offline":[51,362],"setting?We":[52],"employ":[53],"a":[54,67,78,134,326,367,394,411,437,457,493],"two-stage":[55],"training":[56,449],"recipe:":[57],"first,":[58],"Supervised":[59],"Fine-Tuning":[60],"(SFT)":[61],"on":[62,114,127,137,165,180,479],"traces,":[64,446],"followed":[65],"by":[66,195],"refinement":[68],"stage":[69],"using":[70],"traces.We":[75,489],"find":[76],"simple":[79,458],"REINFORCE-style":[80,415],"objective,":[81,89],"which":[82,287,428],"we":[83,306,310,334,398],"term":[84],"Reinforcement":[86,425],"Distillation":[87,426],"(REDI)":[88],"outperforms":[90],"established":[91,340],"preference":[92,341],"optimization":[93,342],"methods":[94,502],"like":[95,503],"DPO":[96],"SimPO":[98,355],"this":[100,109,332,361,430],"context.Our":[102],"empirical":[103],"evaluations":[104],"demonstrate":[105],"effectiveness":[107],"of":[108,132,147,297,339,421,482,501],"approach.Notably,":[110],"our":[111,422],"Qwen-REDI-1.5Bmodel,":[112],"trained":[113,136],"just":[115,166],"131k":[116,167],"open":[120,168,329],"Open-R1":[121],"dataset,":[122],"achieves":[123,171],"83.1%":[125],"score":[126],"MATH-500.Its":[128],"matches":[130],"DeepSeek-R1-Distill-Qwen-1.5B,":[133],"800k":[138,181],"proprietary":[139,182],"result":[141],"showcases":[142],"remarkable":[144,190],"efficiency":[146],"utilizing":[148,197],"previously":[149],"discarded":[150],"traces.*":[152],"Work":[153],"done":[154],"during":[155],"internship":[157],"at":[158],"INFLY":[159],"TECH.trained":[160],"with":[161,325],"REDI":[163,447],"recipe":[164],"points,":[170],"highest":[173],"average":[174],"score.This":[175],"surpasses":[177],"DeepSeek-R1-Distill-Qwen-1.5B":[178,229],"(trained":[179],"data)":[183],"(DeepSeek-AI":[184],"et":[185,351,357],"al.,":[186,352,358],"2025),":[187],"demonstrating":[188],"REDI's":[189],"efficiency.REDI":[192],"enhances":[193],"examples.Values":[203],"bold":[205],"indicate":[206],"best":[208],"each":[211],"column.*":[212],"Officially":[213],"reported":[214],"pass@1":[215],"results.Model":[216],"MATH-500":[217],"AIME24":[218],"AMC23":[219],"Minerva":[220],"OlympiadBench":[221],"Avg.Qwen2.5-Math-1.5B-Instruct*":[222],"75.8":[223],"10.0":[224],"60.0":[225],"29.4":[226],"38.1":[227],"42.7":[228],"83.2":[230],"\u00b10.2":[231,266],"28.3":[232],"\u00b11.0":[233],"62.1":[234],"\u00b10.8":[235],"26.0":[236],"\u00b10.4":[237,252],"43.1":[238],"\u00b10.3":[239,241,246,254,256,258,264,268],"48.6":[240],"Qwen-SFT-1.5B-5ep":[242],"(SFT":[243],"Baseline)":[244],"80.4":[245],"21.9":[247],"\u00b11.1":[248,260],"57.5":[249],"\u00b10.6":[250,262],"27.5":[251],"41.5":[253],"45.8":[255],"Qwen-REDI-1.5B83.1":[257],"28.1":[259],"62.4":[261],"28.8":[263],"45.2":[265],"49.5":[267],"are":[269],"typically":[270],"underutilized.We":[271],"hypothesize":[272],"these":[274,406],"contain":[277],"vital":[278],"insights":[279],"into":[280],"common":[281],"pitfalls":[282],"nuanced":[284],"errors":[285],"could":[290],"learn,":[291],"thereby":[292],"further":[293],"unlocking":[294],"potential":[296],"leads":[300],"central":[303],"research":[304],"question":[305],"address:":[307],"leverage":[312],"fixed":[327],"dataset?To":[330],"address":[331],"challenge,":[333],"first":[335,476],"investigate":[336],"application":[338],"methods,":[343],"such":[344],"as":[345],"Direct":[346],"Preference":[347],"Optimization":[348],"(DPO)":[349],"(Rafailov":[350],"2024)":[353],"(Meng":[356],"2024),":[359],"setting.Our":[364],"analysis":[365],"reveals":[366],"performance-stability":[369,494],"trade-off:":[370],"while":[371],"Kullback-Leibler":[373],"(KL)":[374],"divergence":[375],"penalty":[376],"is":[377],"essential":[378],"for":[379,462],"stable":[380],"training,":[381],"it":[382],"simultaneously":[383],"constrains":[384],"model's":[386],"peak":[387],"achievable":[388],"2":[390],".This":[391],"discovery":[392],"motivated":[393],"deeper":[395],"analysis,":[396],"where":[397],"found":[399],"\u2192":[403],"0":[404],"limit,":[405],"complex":[407],"objectives":[408],"converge":[409],"simpler,":[412],"reference-free":[413],"objective.This":[416],"insight":[417],"forms":[418],"basis":[420],"approach.We":[423],"propose":[424],"(REDI),":[427],"adopts":[429],"performant":[431],"but":[432],"unstable":[433],"objective":[434],"introduces":[436],"asymmetric":[438],"weighting":[439],"scheme.By":[440],"down-weighting":[441],"gradient":[443],"restores":[448],"stability":[450],"without":[451],"sacrificing":[452],"benefits,":[455],"offering":[456],"effective":[460],"method":[461],"learning":[463],"signals.Our":[469],"key":[470],"contributions":[471],"are:":[472],"1.We":[473],"provide":[474],"systematic":[477],"study":[478],"utilization":[481],"correct":[484],"identify":[490],"analyze":[492],"trade-off":[495],"inherent":[496],"KL":[499],"regularization":[500],"DPO,":[504]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2025-10-10T00:00:00"}