{"id":"https://openalex.org/W7161577320","doi":"https://doi.org/10.48550/arxiv.2605.15239","title":"Reducing the Safety Tax in LLM Safety Alignment with On-Policy Self-Distillation","display_name":"Reducing the Safety Tax in LLM Safety Alignment with On-Policy Self-Distillation","publication_year":2026,"publication_date":"2026-05-14","ids":{"openalex":"https://openalex.org/W7161577320","doi":"https://doi.org/10.48550/arxiv.2605.15239"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.15239","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.15239","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.15239","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136383303","display_name":"Yu Fu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136373062","display_name":"Longxuan Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Longxuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136387738","display_name":"Haz Sameen Shahgir","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shahgir, Haz Sameen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136363735","display_name":"Zhipeng Wei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Zhipeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136417537","display_name":"Hui Liu","orcid":"https://orcid.org/0000-0001-5828-0306"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Hui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136385151","display_name":"N. Benjamin Erichson","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Erichson, N. Benjamin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136409952","display_name":"Yue Dong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong, Yue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.33550000190734863,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.33550000190734863,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.09529999643564224,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13295","display_name":"Safety Systems Engineering in Autonomy","score":0.07039999961853027,"subfield":{"id":"https://openalex.org/subfields/2213","display_name":"Safety, Risk, Reliability and Quality"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/safer","display_name":"SAFER","score":0.8382999897003174},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5569999814033508},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.5422999858856201},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.41850000619888306},{"id":"https://openalex.org/keywords/train","display_name":"Train","score":0.33629998564720154},{"id":"https://openalex.org/keywords/data-integrity","display_name":"Data integrity","score":0.3107999861240387}],"concepts":[{"id":"https://openalex.org/C2776654903","wikidata":"https://www.wikidata.org/wiki/Q2601463","display_name":"SAFER","level":2,"score":0.8382999897003174},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6699000000953674},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5569999814033508},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5422999858856201},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.49079999327659607},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.41850000619888306},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3418000042438507},{"id":"https://openalex.org/C190839683","wikidata":"https://www.wikidata.org/wiki/Q2448197","display_name":"Train","level":2,"score":0.33629998564720154},{"id":"https://openalex.org/C33762810","wikidata":"https://www.wikidata.org/wiki/Q461671","display_name":"Data integrity","level":2,"score":0.3107999861240387},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.30730000138282776},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.2874999940395355},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.28110000491142273},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.2743000090122223},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.26499998569488525}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.15239","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.15239","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.15239","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.15239","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.7675102353096008}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Safety":[0],"alignment":[1],"often":[2,123],"improves":[3],"robustness":[4],"to":[5,137],"harmful":[6],"queries":[7],"at":[8],"the":[9,18,30,109,179],"cost":[10],"of":[11,63,94],"reasoning":[12,145],"ability,":[13],"a":[14,60,90,98,118,124,162,217],"tradeoff":[15,165],"known":[16],"as":[17,59],"safety":[19,34,71,100,144,221],"tax.":[20],"A":[21],"common":[22],"cause":[23],"is":[24],"distributional":[25],"mismatch:":[26],"supervised":[27],"fine-tuning":[28],"trains":[29],"target":[31],"model":[32,78,158],"on":[33,47,97,182,187,192],"demonstrations":[35],"produced":[36],"by":[37],"humans,":[38],"external":[39],"models,":[40],"or":[41],"fixed":[42],"self-generated":[43],"traces,":[44],"rather":[45,146],"than":[46,108,147,166],"trajectories":[48],"sampled":[49,110],"from":[50,89],"its":[51,80],"own":[52,81],"policy.":[53],"We":[54,133],"identify":[55],"off-policy":[56,167],"training":[57],"mismatch":[58],"second":[61],"source":[62],"this":[64,103,135],"tax":[65],"and":[66,83,156,169,175,189,200],"study":[67],"on-policy":[68],"self-distillation":[69,168],"for":[70,139,219],"alignment,":[72],"which":[73],"we":[74,113],"call":[75],"OPSA.":[76],"The":[77,194],"generates":[79],"rollouts":[82],"receives":[84],"dense":[85],"per-token":[86],"KL":[87],"supervision":[88],"frozen":[91],"teacher":[92,104],"copy":[93],"itself":[95],"conditioned":[96],"privileged":[99,125],"context.":[101],"Because":[102],"must":[105],"be":[106],"safer":[107],"student":[111],"trajectory,":[112],"introduce":[114],"\\emph{teacher":[115],"flip":[116],"rate}:":[117],"criterion":[119],"that":[120,141,208],"measures":[121],"how":[122],"context":[126],"converts":[127],"unsafe":[128],"responses":[129],"into":[130],"safe":[131],"ones.":[132],"use":[134],"signal":[136],"search":[138],"contexts":[140],"activate":[142],"latent":[143],"merely":[148],"elicit":[149],"safe-looking":[150],"demonstrations.":[151],"Across":[152],"two":[153],"reasoning-model":[154],"families":[155],"five":[157],"scales,":[159],"OPSA":[160,209],"achieves":[161],"stronger":[163],"safety--reasoning":[164],"external-teacher":[170],"distillation":[171],"under":[172],"matched":[173],"data":[174],"full-parameter":[176],"fine-tuning,":[177],"with":[178],"largest":[180],"gains":[181,195],"smaller":[183],"models":[184],"(+8.85":[185],"points":[186,191],"R1-Distill-1.5B":[188],"+5.49":[190],"Qwen3-0.6B).":[193],"persist":[196],"across":[197],"training-set":[198],"sizes":[199],"adaptive":[201],"jailbreak":[202],"evaluations.":[203],"Token-level":[204],"analyses":[205],"further":[206],"show":[207],"concentrates":[210],"updates":[211],"near":[212],"early":[213],"compliance-decision":[214],"tokens,":[215],"providing":[216],"mechanism":[218],"improving":[220],"while":[222],"preserving":[223],"general":[224],"reasoning.":[225]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-19T00:00:00"}