{"id":"https://openalex.org/W7160918678","doi":"https://doi.org/10.48550/arxiv.2605.10194","title":"TRACE: Distilling Where It Matters via Token-Routed Self On-Policy Alignment","display_name":"TRACE: Distilling Where It Matters via Token-Routed Self On-Policy Alignment","publication_year":2026,"publication_date":"2026-05-11","ids":{"openalex":"https://openalex.org/W7160918678","doi":"https://doi.org/10.48550/arxiv.2605.10194"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.10194","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.10194","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.10194","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135972059","display_name":"Jiaxuan Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jiaxuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135970350","display_name":"Xuan Ouyang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ouyang, Xuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135912626","display_name":"Zhiyu Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Zhiyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135987582","display_name":"Yulan Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Yulan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135921444","display_name":"Zheng Pan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pan, Zheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135965284","display_name":"Xin Li","orcid":"https://orcid.org/0000-0001-8927-2149"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Xin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135987369","display_name":"Lan-Zhe Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Lan-Zhe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.376800000667572,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.376800000667572,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.08780000358819962,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.07580000162124634,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/trace","display_name":"TRACE (psycholinguistics)","score":0.6078000068664551},{"id":"https://openalex.org/keywords/kullback\u2013leibler-divergence","display_name":"Kullback\u2013Leibler divergence","score":0.5252000093460083},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.5026000142097473},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4478999972343445},{"id":"https://openalex.org/keywords/verifiable-secret-sharing","display_name":"Verifiable secret sharing","score":0.4275999963283539},{"id":"https://openalex.org/keywords/channel","display_name":"Channel (broadcasting)","score":0.4198000133037567},{"id":"https://openalex.org/keywords/lift","display_name":"Lift (data mining)","score":0.39579999446868896}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6104999780654907},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.6078000068664551},{"id":"https://openalex.org/C171752962","wikidata":"https://www.wikidata.org/wiki/Q255166","display_name":"Kullback\u2013Leibler divergence","level":2,"score":0.5252000093460083},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.5026000142097473},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4478999972343445},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.4275999963283539},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.4198000133037567},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.40470001101493835},{"id":"https://openalex.org/C139002025","wikidata":"https://www.wikidata.org/wiki/Q3001212","display_name":"Lift (data mining)","level":2,"score":0.39579999446868896},{"id":"https://openalex.org/C116188536","wikidata":"https://www.wikidata.org/wiki/Q17118377","display_name":"Swift","level":2,"score":0.32679998874664307},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.32510000467300415},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.28870001435279846},{"id":"https://openalex.org/C88626702","wikidata":"https://www.wikidata.org/wiki/Q1128903","display_name":"Continuation","level":2,"score":0.28360000252723694},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.26980000734329224},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.2680000066757202},{"id":"https://openalex.org/C2777402240","wikidata":"https://www.wikidata.org/wiki/Q6783436","display_name":"Masking (illustration)","level":2,"score":0.2671000063419342},{"id":"https://openalex.org/C91602232","wikidata":"https://www.wikidata.org/wiki/Q756115","display_name":"Volatility (finance)","level":2,"score":0.2606000006198883},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.2597000002861023},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.25519999861717224},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.25029999017715454}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.10194","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.10194","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.10194","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.10194","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"On-policy":[0],"self-distillation":[1],"(self-OPD)":[2],"densifies":[3],"reinforcement":[4],"learning":[5],"with":[6,89],"verifiable":[7],"rewards":[8],"(RLVR)":[9],"by":[10,139],"letting":[11],"a":[12,96],"policy":[13],"teach":[14],"itself":[15],"under":[16,163],"privileged":[17],"context.":[18],"We":[19,53],"find":[20],"that":[21,114,178],"when":[22],"this":[23],"guidance":[24],"spans":[25,72],"the":[26,90,115,147,172,176,187],"full":[27],"response,":[28],"all-token":[29,157],"KL":[30,69,78,91,107,198,209],"spends":[31],"gradients":[32],"on":[33,64,70,79,85,143,152,193,199,203,210],"mostly":[34],"redundant":[35],"positions":[36],"and":[37,46,83,121,145,156],"amplifies":[38],"privileged-information":[39],"leakage,":[40],"causing":[41],"entropy":[42],"rise,":[43],"shortened":[44],"reasoning,":[45],"out-of-distribution":[47],"degradation":[48],"in":[49],"long-horizon":[50],"math":[51,131],"training.":[52],"propose":[54],"Token-Routed":[55],"Alignment":[56],"for":[57],"Critical":[58],"rEasoning":[59],"(TRACE),":[60],"which":[61],"distills":[62],"only":[63],"annotator-marked":[65],"critical":[66],"spans:":[67],"forward":[68,106,197],"key":[71,200],"of":[73,171],"correct":[74],"rollouts,":[75],"optional":[76],"reverse":[77,208],"localized":[80],"error":[81,211],"spans,":[82,201],"GRPO":[84,138,155],"all":[86],"remaining":[87],"tokens,":[88],"channel":[92],"annealed":[93],"away":[94],"after":[95],"short":[97],"warm-up.":[98],"Our":[99],"analysis":[100],"explains":[101],"TRACE":[102,135,179],"through":[103],"two":[104],"effects:":[105],"provides":[108],"non-vanishing":[109],"lift":[110],"to":[111,207],"teacher-supported":[112],"tokens":[113],"student":[116],"under-allocates,":[117],"while":[118,202],"span":[119],"masking":[120],"decay":[122],"keep":[123],"cumulative":[124],"privileged-gradient":[125],"exposure":[126],"finite.":[127],"On":[128],"four":[129],"held-out":[130],"benchmarks":[132],"plus":[133],"GPQA-Diamond,":[134,153],"improves":[136],"over":[137],"2.76":[140],"percentage":[141,167],"points":[142],"average":[144],"preserves":[146],"Qwen3-8B":[148,194],"base":[149],"OOD":[150],"score":[151],"where":[154],"self-OPD":[158],"baselines":[159],"degrade.":[160],"Gains":[161],"persist":[162],"online":[164],"self-annotation":[165],"(+1.90":[166],"points,":[168],"about":[169],"69%":[170],"strong-API":[173],"gain),":[174],"reducing":[175],"concern":[177],"merely":[180],"imports":[181],"external":[182],"annotator":[183],"capability.":[184],"Across":[185],"scales,":[186],"best":[188],"routed":[189],"action":[190],"is":[191,196],"base-dependent:":[192],"it":[195,205],"Qwen3-1.7B":[204],"shifts":[206],"spans.":[212]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-13T00:00:00"}