{"id":"https://openalex.org/W7127425149","doi":"https://doi.org/10.48550/arxiv.2602.01219","title":"Mixture-of-Top-k Attention: Efficient Attention via Scalable Fast Weights","display_name":"Mixture-of-Top-k Attention: Efficient Attention via Scalable Fast Weights","publication_year":2026,"publication_date":"2026-02-01","ids":{"openalex":"https://openalex.org/W7127425149","doi":"https://doi.org/10.48550/arxiv.2602.01219"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.01219","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5124944669","display_name":"Qishuai Wen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wen, Qishuai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104300294","display_name":"Zhiyuan Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Zhiyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102674034","display_name":"Xianghan Meng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Meng, Xianghan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124912182","display_name":"Wei He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5100654349","display_name":"Chunguang Li","orcid":"https://orcid.org/0000-0003-3147-1553"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Chun-Guang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.13422911,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.2451000064611435,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.2451000064611435,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.24390000104904175,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.17080000042915344,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.717199981212616},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.6351000070571899},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.6054999828338623},{"id":"https://openalex.org/keywords/landmark","display_name":"Landmark","score":0.5559999942779541},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.544700026512146},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5177000164985657}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7208999991416931},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.717199981212616},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.6351000070571899},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.6054999828338623},{"id":"https://openalex.org/C2780297707","wikidata":"https://www.wikidata.org/wiki/Q4895393","display_name":"Landmark","level":2,"score":0.5559999942779541},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.544700026512146},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5177000164985657},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.5101000070571899},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.42739999294281006},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4036000072956085},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3815000057220459},{"id":"https://openalex.org/C17020691","wikidata":"https://www.wikidata.org/wiki/Q139677","display_name":"Operator (biology)","level":5,"score":0.3671000003814697},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.3598000109195709},{"id":"https://openalex.org/C88626702","wikidata":"https://www.wikidata.org/wiki/Q1128903","display_name":"Continuation","level":2,"score":0.323199987411499},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.3174000084400177}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.01219","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.01219","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.01219","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.01219","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"vanilla":[1],"self-attention":[2],"mechanism":[3],"in":[4],"Transformers":[5],"can":[6],"be":[7],"viewed":[8],"as":[9,73,100,137,169,171,207],"a":[10,91,114,125,149],"two-layer":[11],"fast-weight":[12,55,74,167],"MLP,":[13],"whose":[14,22],"weights":[15,103],"are":[16],"dynamically":[17],"induced":[18],"by":[19],"inputs":[20],"and":[21,76,110,139,196,201,212],"hidden":[23],"dimension":[24],"is":[25,219],"equal":[26],"to":[27,81,90,131,165,181],"the":[28,33,36,59,66,79,145,157,172,193],"sequence":[29,67],"length":[30],"$N$.":[31],"As":[32],"context":[34],"extends,":[35],"expressive":[37],"capacity":[38],"of":[39,128,159,174,198],"such":[40,206],"an":[41,208],"$N$-width":[42,146],"MLP":[43,147],"increases,":[44],"but":[45],"it":[46],"becomes":[47],"unscalable":[48],"for":[49,94],"extremely":[50],"long":[51],"sequences.":[52],"Recently,":[53],"this":[54,84,88],"perspective":[56,89],"has":[57],"motivated":[58],"Mixture-of-Experts":[60],"(MoE)":[61],"attention":[62,96,162,177],"mechanism,":[63],"which":[64,123],"partitions":[65],"into":[68,113,148],"rigid":[69,164],"blocks,":[70],"treats":[71],"them":[72,99,112],"experts,":[75,142,168],"sparsely":[77],"routes":[78],"tokens":[80],"them.":[82],"In":[83],"paper,":[85],"we":[86,118],"elevate":[87],"unifying":[92],"framework":[93],"efficient":[95],"mechanisms,":[97],"interpreting":[98],"making":[101],"fast":[102],"scalable":[104],"through":[105],"either":[106],"routing":[107],"or":[108],"compression,":[109],"organizing":[111],"five-dimensional":[115],"taxonomy.":[116],"Then,":[117],"propose":[119],"Mixture-of-Top-$k$":[120],"Attention":[121],"(MiTA),":[122],"employs":[124],"small":[126],"set":[127,180],"landmark":[129],"queries":[130],"gather":[132],"top-$k$":[133,176,183],"attended":[134],"key-value":[135],"pairs":[136],"query-aware":[138],"deformable":[140,166],"routed":[141],"while":[143],"compressing":[144],"narrower":[150],"shared":[151],"expert.":[152],"Consequently,":[153],"our":[154,199],"MiTA":[155],"improves":[156],"flexibility":[158],"prior":[160,175],"MoE":[161],"from":[163,178,215],"well":[170],"scalability":[173],"query-specific":[179],"reusable":[182],"set.":[184],"We":[185],"conduct":[186],"extensive":[187],"experiments":[188],"on":[189],"vision":[190],"tasks":[191],"showing":[192],"superior":[194],"effectiveness":[195],"efficiency":[197],"MiTA,":[200],"also":[202],"uncovering":[203],"intriguing":[204],"properties":[205],"emergent":[209],"token-pruning":[210],"effect":[211],"easy":[213],"generalization":[214],"standard":[216],"attention.":[217],"Code":[218],"available":[220],"at":[221],"https://github.com/QishuaiWen/MiTA.":[222]},"counts_by_year":[],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2026-02-04T00:00:00"}