{"id":"https://openalex.org/W7140680812","doi":"https://doi.org/10.48550/arxiv.2603.23516","title":"MSA: Memory Sparse Attention for Efficient End-to-End Memory Model Scaling to 100M Tokens","display_name":"MSA: Memory Sparse Attention for Efficient End-to-End Memory Model Scaling to 100M Tokens","publication_year":2026,"publication_date":"2026-03-06","ids":{"openalex":"https://openalex.org/W7140680812","doi":"https://doi.org/10.48550/arxiv.2603.23516"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.23516","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.23516","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.23516","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5130709198","display_name":"Yu Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129827112","display_name":"Runkai Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Runkai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130697982","display_name":"Sheng Yi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yi, Sheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130630123","display_name":"Xinda Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Xinda","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130686537","display_name":"Xiaohong Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Xiaohong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055260255","display_name":"Jianjin Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Jianjin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130656644","display_name":"Jun Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Jun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063098561","display_name":"Chuanrui Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Chuanrui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130648139","display_name":"Yunyun Han","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Yunyun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130648954","display_name":"Lidong Bing","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bing, Lidong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130686394","display_name":"Yafeng Deng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Deng, Yafeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5130652967","display_name":"Tianqiao Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Tianqiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.12860000133514404,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.12860000133514404,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.11980000138282776,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.11890000104904175,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/interleaved-memory","display_name":"Interleaved memory","score":0.5968000292778015},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.5920000076293945},{"id":"https://openalex.org/keywords/memory-map","display_name":"Memory map","score":0.505299985408783},{"id":"https://openalex.org/keywords/flat-memory-model","display_name":"Flat memory model","score":0.4805999994277954},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.46459999680519104},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.44190001487731934},{"id":"https://openalex.org/keywords/cache-only-memory-architecture","display_name":"Cache-only memory architecture","score":0.4350999891757965},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.40869998931884766},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.40720000863075256},{"id":"https://openalex.org/keywords/memory-model","display_name":"Memory model","score":0.4059999883174896}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8176000118255615},{"id":"https://openalex.org/C63511323","wikidata":"https://www.wikidata.org/wiki/Q908936","display_name":"Interleaved memory","level":4,"score":0.5968000292778015},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.5920000076293945},{"id":"https://openalex.org/C74426580","wikidata":"https://www.wikidata.org/wiki/Q719484","display_name":"Memory map","level":3,"score":0.505299985408783},{"id":"https://openalex.org/C57863822","wikidata":"https://www.wikidata.org/wiki/Q905488","display_name":"Flat memory model","level":4,"score":0.4805999994277954},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.46459999680519104},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.44190001487731934},{"id":"https://openalex.org/C3720319","wikidata":"https://www.wikidata.org/wiki/Q5015937","display_name":"Cache-only memory architecture","level":5,"score":0.4350999891757965},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.40869998931884766},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.40720000863075256},{"id":"https://openalex.org/C12186640","wikidata":"https://www.wikidata.org/wiki/Q6815743","display_name":"Memory model","level":3,"score":0.4059999883174896},{"id":"https://openalex.org/C133371097","wikidata":"https://www.wikidata.org/wiki/Q868014","display_name":"Non-uniform memory access","level":5,"score":0.40130001306533813},{"id":"https://openalex.org/C93446704","wikidata":"https://www.wikidata.org/wiki/Q449328","display_name":"Registered memory","level":3,"score":0.3926999866962433},{"id":"https://openalex.org/C171675096","wikidata":"https://www.wikidata.org/wiki/Q1143380","display_name":"Extended memory","level":4,"score":0.38769999146461487},{"id":"https://openalex.org/C30390489","wikidata":"https://www.wikidata.org/wiki/Q4680748","display_name":"Adaptive memory","level":3,"score":0.3741999864578247},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3635999858379364},{"id":"https://openalex.org/C153247305","wikidata":"https://www.wikidata.org/wiki/Q835713","display_name":"Memory address","level":3,"score":0.362199991941452},{"id":"https://openalex.org/C190475519","wikidata":"https://www.wikidata.org/wiki/Q544384","display_name":"Massively parallel","level":2,"score":0.3587999939918518},{"id":"https://openalex.org/C176649486","wikidata":"https://www.wikidata.org/wiki/Q2308807","display_name":"Memory management","level":3,"score":0.35569998621940613},{"id":"https://openalex.org/C87907426","wikidata":"https://www.wikidata.org/wiki/Q6815755","display_name":"Memory refresh","level":4,"score":0.32850000262260437},{"id":"https://openalex.org/C100800780","wikidata":"https://www.wikidata.org/wiki/Q1175867","display_name":"Memory controller","level":3,"score":0.32100000977516174},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3181999921798706},{"id":"https://openalex.org/C48455012","wikidata":"https://www.wikidata.org/wiki/Q2892593","display_name":"Memory consolidation","level":3,"score":0.3181000053882599},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.31520000100135803},{"id":"https://openalex.org/C98986596","wikidata":"https://www.wikidata.org/wiki/Q1143031","display_name":"Semiconductor memory","level":2,"score":0.3149000108242035},{"id":"https://openalex.org/C2779602883","wikidata":"https://www.wikidata.org/wiki/Q15544750","display_name":"Memory architecture","level":2,"score":0.3147999942302704},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.30970001220703125},{"id":"https://openalex.org/C39528615","wikidata":"https://www.wikidata.org/wiki/Q1229610","display_name":"Distributed shared memory","level":5,"score":0.30720001459121704},{"id":"https://openalex.org/C51290061","wikidata":"https://www.wikidata.org/wiki/Q1936765","display_name":"Uniform memory access","level":4,"score":0.2989000082015991},{"id":"https://openalex.org/C118702147","wikidata":"https://www.wikidata.org/wiki/Q189396","display_name":"Dynamic random-access memory","level":3,"score":0.2953000068664551},{"id":"https://openalex.org/C82687282","wikidata":"https://www.wikidata.org/wiki/Q66221","display_name":"Auxiliary memory","level":2,"score":0.2937999963760376},{"id":"https://openalex.org/C189930140","wikidata":"https://www.wikidata.org/wiki/Q1112878","display_name":"CAS latency","level":4,"score":0.2897000014781952},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.2856999933719635},{"id":"https://openalex.org/C92855701","wikidata":"https://www.wikidata.org/wiki/Q5830907","display_name":"Computer memory","level":3,"score":0.2856000065803528},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.2840000092983246},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.28130000829696655},{"id":"https://openalex.org/C152890283","wikidata":"https://www.wikidata.org/wiki/Q4129922","display_name":"Computing with Memory","level":5,"score":0.2784000039100647},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.2623000144958496},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2605000138282776},{"id":"https://openalex.org/C188045654","wikidata":"https://www.wikidata.org/wiki/Q17148339","display_name":"Memory bandwidth","level":2,"score":0.2599000036716461}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.23516","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.23516","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.23516","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.23516","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Long-term":[0],"memory":[1,51,90,114,132,196,208,219],"is":[2,37],"a":[3,15,93,225],"cornerstone":[4],"of":[5,25,32,95],"human":[6],"intelligence.":[7],"Enabling":[8],"AI":[9],"to":[10,22,40,65,87,167,189,228],"process":[11],"lifetime-scale":[12,234],"information":[13],"remains":[14],"long-standing":[16],"pursuit":[17],"in":[18,149,210],"the":[19,23,28],"field.":[20],"Due":[21],"constraints":[24],"full-attention":[26],"architectures,":[27],"effective":[29],"context":[30,82],"length":[31,83],"large":[33],"language":[34],"models":[35,231],"(LLMs)":[36],"typically":[38],"limited":[39],"1M":[41],"tokens.":[42,169],"Existing":[43],"approaches,":[44],"such":[45],"as":[46,81],"hybrid":[47],"linear":[48,147],"attention,":[49],"fixed-size":[50],"states":[52],"(e.g.,":[53],"RNNs),":[54],"and":[55,77,108,116,129,142,152,206],"external":[56],"storage":[57],"methods":[58],"like":[59,103],"RAG":[60,204],"or":[61,92],"agent":[62,110],"systems,":[63,205],"attempt":[64],"extend":[66],"this":[67],"limit.":[68],"However,":[69],"they":[70],"often":[71],"suffer":[72],"from":[73,165,221],"severe":[74],"precision":[75],"degradation":[76,162],"rapidly":[78],"increasing":[79],"latency":[80],"grows,":[84],"an":[85,125],"inability":[86],"dynamically":[88],"modify":[89],"content,":[91],"lack":[94],"end-to-end":[96,126],"optimization.":[97],"These":[98,213],"bottlenecks":[99],"impede":[100],"complex":[101,191],"scenarios":[102],"large-corpus":[104],"summarization,":[105],"Digital":[106],"Twins,":[107],"long-history":[109],"reasoning,":[111,222],"while":[112,154],"limiting":[113],"capacity":[115,220],"slowing":[117],"inference.":[118],"We":[119,184],"present":[120],"Memory":[121,176,187],"Sparse":[122],"Attention":[123],"(MSA),":[124],"trainable,":[127],"efficient,":[128],"massively":[130],"scalable":[131,139,226],"model":[133],"framework.":[134],"Through":[135],"core":[136],"innovations":[137],"including":[138],"sparse":[140],"attention":[141],"document-wise":[143],"RoPE,":[144],"MSA":[145,198,223],"achieves":[146],"complexity":[148],"both":[150],"training":[151],"inference":[153,180],"maintaining":[155],"exceptional":[156],"stability,":[157],"exhibiting":[158],"less":[159],"than":[160],"9%":[161],"when":[163],"scaling":[164],"16K":[166],"100M":[168],"Furthermore,":[170],"KV":[171],"cache":[172],"compression,":[173],"combined":[174],"with":[175,232],"Parallel,":[177],"enables":[178],"100M-token":[179],"on":[181],"2xA800":[182],"GPUs.":[183],"also":[185],"propose":[186],"Interleaving":[188],"facilitate":[190],"multi-hop":[192],"reasoning":[193],"across":[194],"scattered":[195],"segments.":[197],"significantly":[199],"surpasses":[200],"frontier":[201],"LLMs,":[202],"state-of-the-art":[203],"leading":[207],"agents":[209],"long-context":[211],"benchmarks.":[212],"results":[214],"demonstrate":[215],"that":[216],"by":[217],"decoupling":[218],"provides":[224],"foundation":[227],"endow":[229],"general-purpose":[230],"intrinsic,":[233],"memory.":[235]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-03-27T00:00:00"}