{"id":"https://openalex.org/W7164548514","doi":"https://doi.org/10.48550/arxiv.2606.13392","title":"MiniMax Sparse Attention","display_name":"MiniMax Sparse Attention","publication_year":2026,"publication_date":"2026-06-11","ids":{"openalex":"https://openalex.org/W7164548514","doi":"https://doi.org/10.48550/arxiv.2606.13392"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.13392","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.13392","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.13392","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137162882","display_name":"Xunhao Lai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lai, Xunhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138513054","display_name":"Weiqi Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Weiqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138518727","display_name":"Yufeng Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Yufeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134796418","display_name":"Qiaorui Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Qiaorui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138536163","display_name":"Yang Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Yang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138541766","display_name":"Lunbin Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Lunbin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138485650","display_name":"Xiaolong Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Xiaolong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103535545","display_name":"Haohai Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Haohai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058268359","display_name":"Hao Zhu","orcid":"https://orcid.org/0000-0003-4411-0933"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Haichao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138525926","display_name":"Vito Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Vito","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Hu, Jinkai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Jinkai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Li, Jiayao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Jiayao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Gao, Rui","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Rui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Li, Zekun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Zekun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zhu, Songquan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Songquan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zhou, Jingkai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Jingkai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Zhao, Pengyu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Pengyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.3693999946117401,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.3693999946117401,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.24230000376701355,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.050599999725818634,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5554999709129333},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.49059998989105225},{"id":"https://openalex.org/keywords/minimax","display_name":"Minimax","score":0.4496999979019165},{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.4474000036716461},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.42829999327659607},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.4178999960422516},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.40459999442100525},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.3901999890804291},{"id":"https://openalex.org/keywords/softmax-function","display_name":"Softmax function","score":0.3865000009536743}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7886000275611877},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5554999709129333},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.49059998989105225},{"id":"https://openalex.org/C149728462","wikidata":"https://www.wikidata.org/wiki/Q751319","display_name":"Minimax","level":2,"score":0.4496999979019165},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.4474000036716461},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.42829999327659607},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.4178999960422516},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4088999927043915},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.40790000557899475},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.40459999442100525},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40149998664855957},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3901999890804291},{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.3865000009536743},{"id":"https://openalex.org/C190502265","wikidata":"https://www.wikidata.org/wiki/Q17069496","display_name":"MNIST database","level":3,"score":0.38429999351501465},{"id":"https://openalex.org/C56372850","wikidata":"https://www.wikidata.org/wiki/Q1050404","display_name":"Sparse matrix","level":3,"score":0.3743000030517578},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.3626999855041504},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.3564999997615814},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3319000005722046},{"id":"https://openalex.org/C153180980","wikidata":"https://www.wikidata.org/wiki/Q19776675","display_name":"Commit","level":2,"score":0.3271999955177307},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.31709998846054077},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.31459999084472656},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.30709999799728394},{"id":"https://openalex.org/C2777472644","wikidata":"https://www.wikidata.org/wiki/Q16968992","display_name":"Approximate inference","level":3,"score":0.29899999499320984},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.28209999203681946},{"id":"https://openalex.org/C77637269","wikidata":"https://www.wikidata.org/wiki/Q7002051","display_name":"Neural coding","level":2,"score":0.2777999937534332},{"id":"https://openalex.org/C2776372474","wikidata":"https://www.wikidata.org/wiki/Q508291","display_name":"Simplicity","level":2,"score":0.2761000096797943},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.27379998564720154},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.27300000190734863},{"id":"https://openalex.org/C93959086","wikidata":"https://www.wikidata.org/wiki/Q6888345","display_name":"Model selection","level":2,"score":0.26570001244544983},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.2547000050544739},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.250900000333786}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.13392","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.13392","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.13392","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.13392","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Ultra-long-context":[0],"capability":[1],"is":[2,108,196],"becoming":[3],"indispensable":[4],"for":[5,73],"frontier":[6],"LLMs:":[7],"agentic":[8],"workflows,":[9],"repository-scale":[10],"code":[11],"reasoning,":[12],"and":[13,67,105,142,186],"persistent":[14],"memory":[15],"all":[16],"require":[17],"the":[18,32,86,96],"model":[19,156,204],"to":[20,27,114,146],"jointly":[21],"attend":[22],"over":[23,94],"hundreds":[24],"of":[25,29,35,103,121],"thousands":[26],"millions":[28],"tokens,":[30],"yet":[31],"quadratic":[33],"cost":[34],"softmax":[36],"attention":[37,53,93,145,170],"makes":[38],"this":[39],"untenable":[40],"at":[41,174],"deployment":[42],"scale.":[43],"We":[44],"introduce":[45],"MiniMax":[46],"Sparse":[47],"Attention":[48,58],"(MSA),":[49],"a":[50,70,101,118,133,154],"blockwise":[51],"sparse":[52,79,144],"built":[54],"upon":[55],"Grouped":[56],"Query":[57],"(GQA).":[59],"A":[60,200],"lightweight":[61],"Index":[62],"Branch":[63,88],"scores":[64],"key-value":[65],"blocks":[66],"independently":[68],"selects":[69],"Top-k":[71,140],"subset":[72],"each":[74],"GQA":[75,166],"group,":[76],"enabling":[77],"group-specific":[78],"retrieval":[80],"while":[81,167],"maintaining":[82],"efficient":[83],"block-level":[84],"execution;":[85],"Main":[87],"then":[89],"performs":[90,162],"exact":[91],"block-sparse":[92],"only":[95],"selected":[97],"blocks.":[98],"Designed":[99],"around":[100],"principle":[102],"simplicity":[104],"scalability,":[106],"MSA":[107,131,161,182,207],"deliberately":[109],"streamlined,":[110],"making":[111],"it":[112],"straightforward":[113],"deploy":[115],"efficiently":[116],"across":[117],"broad":[119],"range":[120],"GPUs.":[122],"To":[123],"translate":[124],"sparsity":[125],"into":[126],"practical":[127],"speedups,":[128],"we":[129],"co-design":[130],"with":[132,157,165,178],"GPU":[134],"execution":[135],"path":[136],"that":[137],"uses":[138],"exp-free":[139],"selection":[141],"KV-outer":[143],"improve":[147],"tensor-core":[148],"utilization":[149],"under":[150],"block-granular":[151],"access.":[152],"On":[153],"109B-parameter":[155],"native":[158],"multimodal":[159,203],"training,":[160],"on":[163,191],"par":[164],"reducing":[168],"per-token":[169],"compute":[171],"by":[172,206],"28.4x":[173],"1M":[175],"context.":[176],"Paired":[177],"our":[179],"co-designed":[180],"kernel,":[181],"achieves":[183],"14.2x":[184],"prefill":[185],"7.6x":[187],"decoding":[188],"wall-clock":[189],"speedups":[190],"H800.":[192],"Our":[193],"inference":[194],"kernel":[195],"available":[197],"at:":[198,212],"https://github.com/MiniMax-AI/MSA.":[199],"production-grade":[201],"natively":[202],"powered":[205],"has":[208],"been":[209],"publicly":[210],"released":[211],"https://huggingface.co/MiniMaxAI/MiniMax-M3.":[213]},"counts_by_year":[],"updated_date":"2026-07-21T08:15:58.654021","created_date":"2026-06-13T00:00:00"}