{"id":"https://openalex.org/W7138453718","doi":"https://doi.org/10.48550/arxiv.2603.15619","title":"Mixture-of-Depths Attention","display_name":"Mixture-of-Depths Attention","publication_year":2026,"publication_date":"2026-03-16","ids":{"openalex":"https://openalex.org/W7138453718","doi":"https://doi.org/10.48550/arxiv.2603.15619"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.15619","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.15619","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.15619","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5129659945","display_name":"Lianghui Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Lianghui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129719387","display_name":"Yuxin Fang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fang, Yuxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068257496","display_name":"Bencheng Liao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liao, Bencheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129688301","display_name":"Shijie Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Shijie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022657064","display_name":"Tianheng Cheng","orcid":"https://orcid.org/0009-0003-4100-1659"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Tianheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129687019","display_name":"Zilong Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Zilong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129653544","display_name":"Chen Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129691121","display_name":"Lai Wei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Lai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129677415","display_name":"Yutao Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Yutao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129676601","display_name":"Ya Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Ya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129727896","display_name":"Yi Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Yi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129728395","display_name":"Yu Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129663592","display_name":"Xinggang Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xinggang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1906999945640564,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1906999945640564,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.17239999771118164,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.14339999854564667,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perplexity","display_name":"Perplexity","score":0.8568000197410583},{"id":"https://openalex.org/keywords/flops","display_name":"FLOPS","score":0.6452999711036682},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.6097000241279602},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5953999757766724},{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.5321999788284302},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.5289000272750854},{"id":"https://openalex.org/keywords/layer","display_name":"Layer (electronics)","score":0.4862000048160553},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.444599986076355}],"concepts":[{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.8568000197410583},{"id":"https://openalex.org/C3826847","wikidata":"https://www.wikidata.org/wiki/Q188768","display_name":"FLOPS","level":2,"score":0.6452999711036682},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6333000063896179},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.6097000241279602},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5953999757766724},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.5321999788284302},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.5289000272750854},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.4862000048160553},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.444599986076355},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4343000054359436},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.423799991607666},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.414900004863739},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.36970001459121704},{"id":"https://openalex.org/C2776207758","wikidata":"https://www.wikidata.org/wiki/Q5303302","display_name":"Downstream (manufacturing)","level":2,"score":0.3596999943256378},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.34459999203681946},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.3111000061035156},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.29980000853538513},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.27559998631477356},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.2728999853134155},{"id":"https://openalex.org/C2780312720","wikidata":"https://www.wikidata.org/wiki/Q5689100","display_name":"Head (geology)","level":2,"score":0.2711000144481659},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2572999894618988},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.2572000026702881}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.15619","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.15619","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.15619","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.15619","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Scaling":[0],"depth":[1,66,161],"is":[2,156,164],"a":[3,48,75,91,129,157],"key":[4],"driver":[5],"for":[6,78,160],"large":[7],"language":[8],"models":[9,99],"(LLMs).":[10],"Yet,":[11],"as":[12],"LLMs":[13],"become":[14],"deeper,":[15],"they":[16],"often":[17],"suffer":[18],"from":[19,69],"signal":[20],"degradation:":[21],"informative":[22],"features":[23],"formed":[24],"in":[25,40],"shallow":[26],"layers":[27],"are":[28],"gradually":[29],"diluted":[30],"by":[31,112,122],"repeated":[32],"residual":[33],"updates,":[34],"making":[35],"them":[36],"harder":[37],"to":[38,55,57],"recover":[39],"deeper":[41],"layers.":[42,71],"We":[43,72,135],"introduce":[44],"mixture-of-depths":[45],"attention":[46,53],"(MoDA),":[47],"mechanism":[49],"that":[50,80,101,138,154],"allows":[51],"each":[52],"head":[54],"attend":[56],"sequence":[58,92],"KV":[59,67],"pairs":[60,68],"at":[61,90,166],"the":[62],"current":[63],"layer":[64],"and":[65,118],"preceding":[70],"further":[73],"describe":[74],"hardware-efficient":[76],"algorithm":[77],"MoDA":[79,102,140,155],"resolves":[81],"non-contiguous":[82],"memory-access":[83],"patterns,":[84],"achieving":[85],"97.3%":[86],"of":[87,94],"FlashAttention-2's":[88],"efficiency":[89],"length":[93],"64K.":[95],"Experiments":[96],"on":[97,124],"1.5B-parameter":[98],"demonstrate":[100],"consistently":[103],"outperforms":[104],"strong":[105],"baselines.":[106],"Notably,":[107],"it":[108,148],"improves":[109],"average":[110,120],"perplexity":[111],"0.2":[113],"across":[114],"10":[115,125],"validation":[116],"benchmarks":[117],"increases":[119],"performance":[121,145],"2.11%":[123],"downstream":[126],"tasks,":[127],"with":[128,141,149],"negligible":[130],"3.7%":[131],"FLOPs":[132],"computational":[133],"overhead.":[134],"also":[136],"find":[137],"combining":[139],"post-norm":[142],"yields":[143],"better":[144],"than":[146],"using":[147],"pre-norm.":[150],"These":[151],"results":[152],"suggest":[153],"promising":[158],"primitive":[159],"scaling.":[162],"Code":[163],"released":[165],"https://github.com/hustvl/MoDA":[167],".":[168]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-18T00:00:00"}