{"id":"https://openalex.org/W7160849836","doi":"https://doi.org/10.48550/arxiv.2605.07076","title":"Self-Consolidating Language Models: Continual Knowledge Incorporation from Context","display_name":"Self-Consolidating Language Models: Continual Knowledge Incorporation from Context","publication_year":2026,"publication_date":"2026-05-08","ids":{"openalex":"https://openalex.org/W7160849836","doi":"https://doi.org/10.48550/arxiv.2605.07076"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.07076","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.07076","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.07076","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135874125","display_name":"Zekun Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zekun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135878298","display_name":"Anant Gupta","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gupta, Anant","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135843745","display_name":"Zihan Dong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong, Zihan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135832096","display_name":"Christopher J. MacLellan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"MacLellan, Christopher J.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.38109999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.38109999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.09120000153779984,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.07699999958276749,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/limiting","display_name":"Limiting","score":0.6545000076293945},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6237000226974487},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5724999904632568},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5008000135421753},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.4927000105381012},{"id":"https://openalex.org/keywords/data-stream-mining","display_name":"Data stream mining","score":0.39590001106262207},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.3928000032901764}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8319000005722046},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.6545000076293945},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6237000226974487},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5724999904632568},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5382000207901001},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5008000135421753},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.4927000105381012},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4507000148296356},{"id":"https://openalex.org/C89198739","wikidata":"https://www.wikidata.org/wiki/Q3079880","display_name":"Data stream mining","level":2,"score":0.39590001106262207},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3928000032901764},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3882000148296356},{"id":"https://openalex.org/C93959086","wikidata":"https://www.wikidata.org/wiki/Q6888345","display_name":"Model selection","level":2,"score":0.34769999980926514},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.34540000557899475},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.3089999854564667},{"id":"https://openalex.org/C2778484313","wikidata":"https://www.wikidata.org/wiki/Q1172540","display_name":"Data stream","level":2,"score":0.2996000051498413},{"id":"https://openalex.org/C2982912361","wikidata":"https://www.wikidata.org/wiki/Q1851867","display_name":"Mental model","level":2,"score":0.29679998755455017},{"id":"https://openalex.org/C186625053","wikidata":"https://www.wikidata.org/wiki/Q1130191","display_name":"Information overload","level":2,"score":0.2515000104904175}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.07076","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.07076","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.07076","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.07076","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7575283050537109}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"increasingly":[4],"receive":[5],"information":[6,28],"as":[7],"streams":[8,187,192],"of":[9,75,144,163],"passages,":[10],"conversations,":[11],"and":[12,31,116,131,139],"long-context":[13,123],"workflows.":[14],"While":[15],"longer":[16,189],"context":[17,36,40],"windows":[18],"expose":[19],"more":[20],"evidence,":[21],"they":[22],"do":[23],"not":[24],"ensure":[25],"that":[26,89,149,159,168,196],"useful":[27],"is":[29],"preserved":[30],"reused.":[32],"We":[33,51,105],"study":[34],"continual":[35],"consolidation:":[37],"writing":[38],"current":[39,63],"into":[41],"model":[42,88,103,170],"weights":[43],"while":[44,178],"limiting":[45,179],"interference":[46],"with":[47,97,108,161],"previously":[48],"consolidated":[49],"information.":[50],"propose":[52],"\\textbf{S}elf-\\textbf{Co}nsolidating":[53],"\\textbf{L}anguage":[54],"Models":[55],"(SCoL),":[56],"a":[57],"post-training":[58],"framework":[59,198],"in":[60],"which,":[61],"given":[62],"context,":[64],"an":[65,101],"LLM":[66,153],"learns":[67,171],"to":[68,154,172,188],"generate":[69,155],"textual":[70],"update":[71,157],"instructions":[72],"specifying":[73],"which":[74],"its":[76],"own":[77],"Transformer":[78],"layers":[79,162],"should":[80],"be":[81],"updated.":[82],"Because":[83],"committed":[84],"updates":[85],"change":[86],"the":[87,152,169],"later":[90],"generates":[91],"future":[92],"selections,":[93],"we":[94],"train":[95],"SCoL":[96,107,128,150,182],"meta-reinforcement":[98],"learning":[99],"over":[100,133],"evolving":[102],"state.":[104],"instantiate":[106],"supervised":[109],"QA":[110],"rewards":[111,119],"on":[112],"SQuAD":[113],"knowledge":[114],"incorporation":[115],"intrinsic":[117],"likelihood-based":[118],"for":[120],"LongBench":[121,190],"v2":[122,191],"consolidation.":[124,202],"Across":[125],"both":[126],"settings,":[127],"improves":[129],"acquisition":[130],"retention":[132],"prompting,":[134],"summarization,":[135],"batch":[136],"test-time":[137],"training,":[138],"sequential":[140],"finetuning":[141],"baselines.":[142],"Analysis":[143],"learned":[145],"selection":[146],"patterns":[147],"shows":[148],"encourages":[151],"sparse":[156],"locations":[158],"align":[160],"high":[164],"Fisher":[165],"information,":[166],"suggesting":[167,195],"route":[173],"plasticity":[174],"toward":[175],"loss-sensitive":[176],"regions":[177],"interference.":[180],"Moreover,":[181],"transfers":[183],"from":[184],"shorter":[185],"meta-training":[186],"at":[193],"evaluation,":[194],"our":[197],"supports":[199],"scalable":[200],"streaming":[201]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-12T00:00:00"}