{"id":"https://openalex.org/W7161989155","doi":"https://doi.org/10.48550/arxiv.2605.20948","title":"Memory Grafting: Scaling Language Model Pre-training via Offline Conditional Memory","display_name":"Memory Grafting: Scaling Language Model Pre-training via Offline Conditional Memory","publication_year":2026,"publication_date":"2026-05-20","ids":{"openalex":"https://openalex.org/W7161989155","doi":"https://doi.org/10.48550/arxiv.2605.20948"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.20948","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.20948","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.20948","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136013577","display_name":"Runxi Cheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Runxi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136650539","display_name":"Yuchen Guan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guan, Yuchen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136611815","display_name":"Yongxian Wei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Yongxian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104330972","display_name":"Qianpu Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Qianpu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136726895","display_name":"Qixiu Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Qixiu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102598017","display_name":"Sinan Du","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Du, Sinan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136712799","display_name":"Feng Xiong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiong, Feng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136712902","display_name":"Chun Yuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan, Chun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136614182","display_name":"Yan Lu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Yan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136712380","display_name":"Yeyun Gong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gong, Yeyun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.41029998660087585,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.41029998660087585,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1868000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.036400001496076584,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6363999843597412},{"id":"https://openalex.org/keywords/engram","display_name":"Engram","score":0.6064000129699707},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5302000045776367},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.44839999079704285},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.4440000057220459},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.39739999175071716},{"id":"https://openalex.org/keywords/memory-management","display_name":"Memory management","score":0.37880000472068787}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.701200008392334},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6363999843597412},{"id":"https://openalex.org/C28093023","wikidata":"https://www.wikidata.org/wiki/Q175168","display_name":"Engram","level":2,"score":0.6064000129699707},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5302000045776367},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.44839999079704285},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.4440000057220459},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4383000135421753},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.39739999175071716},{"id":"https://openalex.org/C176649486","wikidata":"https://www.wikidata.org/wiki/Q2308807","display_name":"Memory management","level":3,"score":0.37880000472068787},{"id":"https://openalex.org/C12186640","wikidata":"https://www.wikidata.org/wiki/Q6815743","display_name":"Memory model","level":3,"score":0.36820000410079956},{"id":"https://openalex.org/C2775937945","wikidata":"https://www.wikidata.org/wiki/Q222958","display_name":"Grafting","level":3,"score":0.3407000005245209},{"id":"https://openalex.org/C171675096","wikidata":"https://www.wikidata.org/wiki/Q1143380","display_name":"Extended memory","level":4,"score":0.33070001006126404},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.3206999897956848},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.29809999465942383},{"id":"https://openalex.org/C2779804580","wikidata":"https://www.wikidata.org/wiki/Q102047","display_name":"Suffix","level":2,"score":0.2955000102519989},{"id":"https://openalex.org/C82687282","wikidata":"https://www.wikidata.org/wiki/Q66221","display_name":"Auxiliary memory","level":2,"score":0.27320000529289246},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.26980000734329224},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.26249998807907104},{"id":"https://openalex.org/C123593499","wikidata":"https://www.wikidata.org/wiki/Q6008583","display_name":"In-Memory Processing","level":5,"score":0.2599000036716461}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.20948","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.20948","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.20948","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.20948","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Scaling":[0],"conditional":[1,37,51],"memory":[2,19,26,38,69],"offers":[3],"a":[4,36,47,93,208],"promising":[5],"way":[6],"to":[7,119,173],"increase":[8],"language-model":[9],"capacity,":[10],"but":[11],"existing":[12],"methods":[13],"such":[14],"as":[15,50,68,200],"Engram":[16,95,152,172],"learn":[17],"large":[18],"tables":[20],"from":[21,46,164],"scratch":[22],"during":[23],"pre-training,":[24],"making":[25],"scaling":[27,39,212],"expensive":[28],"and":[29,71,90,110,131,139,150,168],"sometimes":[30],"ineffective.":[31],"We":[32],"propose":[33],"Memory":[34,122,144],"Grafting,":[35],"method":[40],"that":[41,143,195],"utilizes":[42],"frozen":[43],"hidden":[44,66],"states":[45],"grafting":[48,61,104],"model":[49,62,75,105],"n-gram":[52],"memory.":[53],"Given":[54],"frequent":[55],"local":[56],"n-grams,":[57],"we":[58],"run":[59,108],"the":[60,73,103,155,160,176,184,189],"offline,":[63],"store":[64],"final-token":[65],"representations":[67],"values,":[70],"let":[72],"recipient":[74,137],"retrieve":[76],"them":[77],"through":[78],"exact":[79,111],"longest-match":[80],"suffix":[81],"lookup.":[82],"Retrieved":[83],"memories":[84],"are":[85],"adapted":[86],"by":[87],"lightweight":[88],"projections":[89],"gates,":[91],"while":[92],"hash-based":[94],"fallback":[96],"preserves":[97],"coverage":[98],"for":[99,166,170],"unmatched":[100],"contexts.":[101],"Since":[102],"is":[106],"only":[107],"offline":[109],"lookup":[112],"has":[113],"expected":[114],"O(1)":[115],"complexity":[116],"with":[117,128,186],"respect":[118],"memory-bank":[120],"size,":[121],"Grafting":[123,145],"expands":[124],"external":[125,204],"latent":[126,205],"capacity":[127],"limited":[129],"training":[130],"inference":[132],"overhead.":[133],"Experiments":[134],"under":[135],"matched":[136],"architectures":[138],"pre-training":[140],"budgets":[141],"show":[142],"improves":[146,159],"over":[147,183],"both":[148],"MoE":[149,167],"vanilla":[151,171],"baselines.":[153],"In":[154,175],"2.8B-scale":[156],"setting,":[157,178],"it":[158],"average":[161],"benchmark":[162],"score":[163],"51.95":[165],"52.43":[169],"53.86.":[174],"0.92B-scale":[177],"all":[179],"grafting-model":[180],"variants":[181],"improve":[182],"baselines,":[185],"Qwen3.5-35B-A3B":[187],"giving":[188],"strongest":[190],"gains.":[191],"These":[192],"results":[193],"suggest":[194],"pretrained":[196],"models":[197,215],"can":[198],"serve":[199],"reusable":[201],"constructors":[202],"of":[203],"memory,":[206],"providing":[207],"practical":[209],"step":[210],"toward":[211],"future":[213],"language":[214],"beyond":[216],"trainable":[217],"parameters":[218],"alone.":[219]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-22T00:00:00"}