{"id":"https://openalex.org/W7162780039","doi":"https://doi.org/10.48550/arxiv.2605.29075","title":"Knowledge Offloading: Decomposing LLMs into Sparse Backbones and Memory Modules","display_name":"Knowledge Offloading: Decomposing LLMs into Sparse Backbones and Memory Modules","publication_year":2026,"publication_date":"2026-05-27","ids":{"openalex":"https://openalex.org/W7162780039","doi":"https://doi.org/10.48550/arxiv.2605.29075"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.29075","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.29075","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.29075","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5097335764","display_name":"Karim Galliamov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Galliamov, Karim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109855717","display_name":"Rochelle Choenni","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Choenni, Rochelle","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137388924","display_name":"Ivan Titov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Titov, Ivan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.26269999146461487,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.26269999146461487,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.23409999907016754,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.131400004029274,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.7329999804496765},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6960999965667725},{"id":"https://openalex.org/keywords/core","display_name":"Core (optical fiber)","score":0.5131999850273132},{"id":"https://openalex.org/keywords/decomposition","display_name":"Decomposition","score":0.5112000107765198},{"id":"https://openalex.org/keywords/encode","display_name":"ENCODE","score":0.4837000072002411},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.47099998593330383}],"concepts":[{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.7329999804496765},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7059000134468079},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6960999965667725},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.5131999850273132},{"id":"https://openalex.org/C124681953","wikidata":"https://www.wikidata.org/wiki/Q339062","display_name":"Decomposition","level":2,"score":0.5112000107765198},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4880000054836273},{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.4837000072002411},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.47099998593330383},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.4433000087738037},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.37369999289512634},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.3521000146865845},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.31189998984336853},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3028999865055084},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.3000999987125397},{"id":"https://openalex.org/C142853389","wikidata":"https://www.wikidata.org/wiki/Q744778","display_name":"Association (psychology)","level":2,"score":0.2727999985218048},{"id":"https://openalex.org/C2777220311","wikidata":"https://www.wikidata.org/wiki/Q6423340","display_name":"Knowledge acquisition","level":2,"score":0.25450000166893005}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.29075","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.29075","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.29075","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.29075","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"LLMs":[0],"encode":[1],"both":[2],"general":[3],"capabilities":[4],"and":[5,55,71,79,85,139,145,180],"domain-specific":[6,56],"knowledge":[7,33,172],"in":[8,26,109,165],"a":[9,27,43,47,51,60,67,106,177],"single":[10],"set":[11],"of":[12,101,120],"parameters.":[13],"We":[14,38],"ask":[15],"whether":[16],"this":[17],"capacity":[18,96],"can":[19,97,173],"be":[20,98,174],"reorganized:":[21],"keeping":[22],"broadly":[23],"useful":[24],"computation":[25],"shared":[28,53,103,178],"backbone,":[29],"while":[30,125,160],"moving":[31],"specialized":[32],"into":[34,50],"external":[35,182],"memory":[36],"modules.":[37],"propose":[39],"\\emph{knowledge":[40],"offloading}":[41],"(KOFF),":[42],"framework":[44],"for":[45],"decomposing":[46],"pretrained":[48],"LLM":[49],"sparse":[52],"backbone":[54,104],"memories.":[57,183],"Starting":[58],"from":[59,88],"frozen":[61,129],"base":[62],"model,":[63],"we":[64,92],"jointly":[65],"learn":[66],"structured":[68],"pruning":[69,126],"mask":[70],"lightweight":[72],"recovery":[73],"modules,":[74],"implemented":[75],"as":[76],"LoRA":[77,138],"adapters":[78],"learned":[80,140,151],"key-value":[81],"caches.":[82],"Across":[83],"Llama":[84],"Qwen":[86],"models":[87],"3B":[89],"to":[90],"8B,":[91],"find":[93],"that":[94,137,149,171],"non-trivial":[95],"moved":[99],"out":[100],"the":[102,121,127,150,166],"without":[105,131],"large":[107],"loss":[108],"model":[110,130],"ability.":[111],"At":[112],"around":[113],"12\\%":[114],"global":[115],"sparsity,":[116],"KOFF":[117],"preserves":[118],"much":[119],"unpruned":[122],"model's":[123],"performance,":[124],"same":[128],"memories":[132,142],"degrades":[133],"sharply.":[134],"Ablations":[135],"show":[136],"KV":[141],"are":[143,157],"complementary,":[144],"specialization":[146],"analyses":[147],"suggest":[148,170],"decomposition":[152],"is":[153],"meaningful:":[154],"language-specific":[155],"neurons":[156,162],"preferentially":[158],"removed":[159],"language-general":[161],"largely":[163],"remain":[164],"backbone.":[167],"These":[168],"results":[169],"reallocated":[175],"between":[176],"core":[179],"swappable":[181]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-30T00:00:00"}