{"id":"https://openalex.org/W7161709916","doi":"https://doi.org/10.48550/arxiv.2605.16865","title":"MixSD: Mixed Contextual Self-Distillation for Knowledge Injection","display_name":"MixSD: Mixed Contextual Self-Distillation for Knowledge Injection","publication_year":2026,"publication_date":"2026-05-16","ids":{"openalex":"https://openalex.org/W7161709916","doi":"https://doi.org/10.48550/arxiv.2605.16865"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.16865","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.16865","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.16865","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136487197","display_name":"Jiarui Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jiarui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136483116","display_name":"Lechen Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Lechen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136463999","display_name":"Yongjin Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Yongjin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136483865","display_name":"Yinghui He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Yinghui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054001148","display_name":"Yingheng Wang","orcid":"https://orcid.org/0000-0002-4828-4757"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yingheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136468686","display_name":"Weihao Xuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xuan, Weihao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136491019","display_name":"Zhijing Jin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, Zhijing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136472274","display_name":"Mona Diab","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Diab, Mona","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8159999847412109,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8159999847412109,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.06700000166893005,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.02710000053048134,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/forgetting","display_name":"Forgetting","score":0.7020999789237976},{"id":"https://openalex.org/keywords/knowledge-base","display_name":"Knowledge base","score":0.6614000201225281},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.5864999890327454},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.579800009727478},{"id":"https://openalex.org/keywords/base","display_name":"Base (topology)","score":0.5216000080108643},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.5109999775886536},{"id":"https://openalex.org/keywords/forcing","display_name":"Forcing (mathematics)","score":0.4699999988079071},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.42820000648498535},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.3790000081062317}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7044000029563904},{"id":"https://openalex.org/C7149132","wikidata":"https://www.wikidata.org/wiki/Q1377840","display_name":"Forgetting","level":2,"score":0.7020999789237976},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.6614000201225281},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5864999890327454},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.582099974155426},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.579800009727478},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.5216000080108643},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.5109999775886536},{"id":"https://openalex.org/C197115733","wikidata":"https://www.wikidata.org/wiki/Q1003136","display_name":"Forcing (mathematics)","level":2,"score":0.4699999988079071},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4596000015735626},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.42820000648498535},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3790000081062317},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.3610000014305115},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3598000109195709},{"id":"https://openalex.org/C132758656","wikidata":"https://www.wikidata.org/wiki/Q5307365","display_name":"Dreyfus model of skill acquisition","level":2,"score":0.33809998631477356},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.33009999990463257},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.32280001044273376},{"id":"https://openalex.org/C2777220311","wikidata":"https://www.wikidata.org/wiki/Q6423340","display_name":"Knowledge acquisition","level":2,"score":0.3215000033378601},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.3066999912261963},{"id":"https://openalex.org/C58328972","wikidata":"https://www.wikidata.org/wiki/Q184609","display_name":"Expert system","level":2,"score":0.29319998621940613},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.29280000925064087},{"id":"https://openalex.org/C115925183","wikidata":"https://www.wikidata.org/wiki/Q1412694","display_name":"Knowledge-based systems","level":2,"score":0.2777999937534332},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.272599995136261},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.2651999890804291},{"id":"https://openalex.org/C159032336","wikidata":"https://www.wikidata.org/wiki/Q2488768","display_name":"Non-monotonic logic","level":2,"score":0.257999986410141},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.2540000081062317},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.2538999915122986}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.16865","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.16865","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.16865","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.16865","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.4903753697872162,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Supervised":[0],"fine-tuning":[1,31],"(SFT)":[2],"is":[3,238],"widely":[4],"used":[5],"to":[6,47,121,136,174,183],"inject":[7],"new":[8],"knowledge":[9,65,158,245],"into":[10],"language":[11],"models,":[12],"but":[13],"it":[14],"often":[15],"degrades":[16],"pretrained":[17],"capabilities":[18],"such":[19],"as":[20,200,202],"reasoning":[21],"and":[22,98,140,157,164,176,218,241],"general-domain":[23],"performance.":[24],"We":[25,126,204],"argue":[26],"this":[27,54],"forgetting":[28],"arises":[29],"because":[30],"targets":[32,213],"from":[33,39,80],"humans":[34],"or":[35],"external":[36],"systems":[37],"diverge":[38],"the":[40,45,84,93,104,113,122,186,215,233],"model's":[41,105,124,188,234],"autoregressive":[42],"distribution,":[43],"forcing":[44],"optimizer":[46],"imitate":[48],"low-probability":[49],"token":[50],"sequences.":[51],"To":[52],"address":[53],"problem,":[55],"we":[56,134],"propose":[57],"MixSD,":[58],"a":[59,99,145,169,239],"simple":[60,240],"external-teacher-free":[61],"method":[62],"for":[63,152,244],"distribution-aligned":[64],"injection.":[66],"Instead":[67],"of":[68,83,185],"training":[69,194],"on":[70,129],"fixed":[71],"targets,":[72],"MixSD":[73,128,166,208],"constructs":[74],"supervision":[75,110,212,231],"dynamically":[76],"by":[77],"mixing":[78],"tokens":[79],"two":[81,130],"conditionals":[82],"base":[85,123,187,216],"model":[86,162,217],"itself:":[87],"an":[88],"expert":[89],"conditional":[90,101],"that":[91,102,133,207,229,247],"observes":[92],"injected":[94],"fact":[95],"in":[96,144],"context,":[97],"naive":[100],"reflects":[103],"original":[106],"prior.":[107],"The":[108],"resulting":[109],"sequences":[111],"preserve":[112],"factual":[114,138,154],"learning":[115],"signal":[116],"while":[117,191],"remaining":[118],"substantially":[119,210],"closer":[120],"distribution.":[125],"evaluate":[127],"synthetic":[131],"corpora":[132],"construct":[135],"study":[137],"recall":[139],"arithmetic":[141],"function":[142],"acquisition":[143],"controlled":[146],"setting,":[147],"together":[148],"with":[149,232],"established":[150],"benchmarks":[151],"open-domain":[153],"question":[155],"answering":[156],"editing.":[159],"Across":[160],"multiple":[161],"scales":[163],"settings,":[165],"consistently":[167],"achieves":[168],"better":[170],"memorization-retention":[171],"trade-off":[172],"compared":[173],"SFT":[175,198],"on-policy":[177],"self":[178],"distillation":[179],"baselines,":[180],"retaining":[181],"up":[182],"100%":[184],"held-out":[189],"capability":[190],"maintaining":[192],"near-perfect":[193],"accuracy,":[195],"whereas":[196],"standard":[197],"retains":[199],"little":[201],"1%.":[203],"further":[205],"show":[206],"produces":[209],"lower-NLL":[211],"under":[214],"reduces":[219],"harmful":[220],"movement":[221],"along":[222],"Fisher-sensitive":[223],"parameter":[224],"directions.":[225],"These":[226],"results":[227],"suggest":[228],"aligning":[230],"native":[235],"generation":[236],"distribution":[237],"effective":[242],"principle":[243],"injection":[246],"mitigates":[248],"catastrophic":[249],"forgetting.":[250]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-20T00:00:00"}