{"id":"https://openalex.org/W7128718552","doi":"https://doi.org/10.48448/gj6k-av52","title":"Making Every Head Count: Sparse Attention Without the Speed-Performance Trade-off","display_name":"Making Every Head Count: Sparse Attention Without the Speed-Performance Trade-off","publication_year":2026,"publication_date":"2026-01-07","ids":{"openalex":"https://openalex.org/W7128718552","doi":"https://doi.org/10.48448/gj6k-av52"},"language":"en","primary_location":{"id":"doi:10.48448/gj6k-av52","doi":"https://doi.org/10.48448/gj6k-av52","is_oa":true,"landing_page_url":"https://doi.org/10.48448/gj6k-av52","pdf_url":null,"source":{"id":"https://openalex.org/S7407053148","display_name":"Underline Science Inc.","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Audiovisual"},"volume":null,"type":"other","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48448/gj6k-av52","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125741655","display_name":"Association for Artificial Intelligence 2026","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Association for Artificial Intelligence 2026","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125757107","display_name":"Wentao Hu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hu, Wentao","raw_affiliation_strings":["Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068378237","display_name":"Tianqi Huang","orcid":"https://orcid.org/0009-0003-4839-9954","observed_orcids":["https://orcid.org/0009-0003-4839-9954","https://orcid.org/0009-0008-4028-8244"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Tianchen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125705878","display_name":"Xin Lai","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lai, Xin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120402342","display_name":"Yuheng Min","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Min, Yuheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125754049","display_name":"Jiayin Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wang, Jiayin","raw_affiliation_strings":["Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141006947","display_name":"Rui Yan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yan, Rui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125773816","display_name":"Mingkuan Zhao","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Mingkuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5125762446","display_name":"Xiaoyan Zhu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhu, Xiaoyan","raw_affiliation_strings":["Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8788999915122986,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8788999915122986,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.05009999871253967,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.042100001126527786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sparse-attention","display_name":"sparse attention","score":0.8930000066757202},{"id":"https://openalex.org/keywords/multi-head-attention","display_name":"multi-head attention","score":0.9860000014305115},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8889999985694885},{"id":"https://openalex.org/keywords/computational-efficiency","display_name":"computational efficiency","score":0.6029999852180481},{"id":"https://openalex.org/keywords/structural-sparsity","display_name":"structural sparsity","score":0.7020000219345093},{"id":"https://openalex.org/keywords/inductive-bias","display_name":"inductive bias","score":0.7289999723434448},{"id":"https://openalex.org/keywords/training-throughput","display_name":"training throughput","score":0.6299999952316284}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7889000177383423},{"id":"https://openalex.org/C179799912","wikidata":"https://www.wikidata.org/wiki/Q205084","display_name":"Computational complexity theory","level":2,"score":0.6449000239372253},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5746999979019165},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5738999843597412},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.5163000226020813},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5131999850273132},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.5065000057220459},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45260000228881836},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4244999885559082},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.39820000529289246},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.39340001344680786},{"id":"https://openalex.org/C127964446","wikidata":"https://www.wikidata.org/wiki/Q1092142","display_name":"Computational resource","level":3,"score":0.3917999863624573},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.38989999890327454},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37770000100135803},{"id":"https://openalex.org/C195956108","wikidata":"https://www.wikidata.org/wiki/Q7268362","display_name":"Quadratic growth","level":2,"score":0.35600000619888306},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.34200000762939453},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.3391999900341034},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.3345000147819519},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.30390000343322754},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3034999966621399},{"id":"https://openalex.org/C2781039887","wikidata":"https://www.wikidata.org/wiki/Q1391724","display_name":"Factor (programming language)","level":2,"score":0.2906000018119812},{"id":"https://openalex.org/C19754495","wikidata":"https://www.wikidata.org/wiki/Q3435924","display_name":"Computational problem","level":2,"score":0.28029999136924744},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.26899999380111694},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.25440001487731934}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48448/gj6k-av52","doi":"https://doi.org/10.48448/gj6k-av52","is_oa":true,"landing_page_url":"https://doi.org/10.48448/gj6k-av52","pdf_url":null,"source":{"id":"https://openalex.org/S7407053148","display_name":"Underline Science Inc.","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Audiovisual"}],"best_oa_location":{"id":"doi:10.48448/gj6k-av52","doi":"https://doi.org/10.48448/gj6k-av52","is_oa":true,"landing_page_url":"https://doi.org/10.48448/gj6k-av52","pdf_url":null,"source":{"id":"https://openalex.org/S7407053148","display_name":"Underline Science Inc.","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Audiovisual"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4415337169","https://openalex.org/W4417339177","https://openalex.org/W7161565130","https://openalex.org/W4416030376","https://openalex.org/W4402599357","https://openalex.org/W4400025310","https://openalex.org/W7135037240","https://openalex.org/W4417466280","https://openalex.org/W7127395297","https://openalex.org/W4403579453"],"abstract_inverted_index":{"The":[0,156],"design":[1,268],"of":[2,29,46,92,154,170,269],"Large":[3],"Language":[4],"Models":[5],"(LLMs)":[6],"has":[7],"long":[8],"been":[9],"hampered":[10],"by":[11,113,151],"a":[12,26,93,127,143,152,166,262],"fundamental":[13],"conflict":[14],"within":[15],"their":[16],"core":[17,87],"attention":[18,61,117,135,226],"mechanism:":[19],"its":[20,205],"remarkable":[21],"expressivity":[22],"is":[23,89,207],"built":[24],"upon":[25],"computational":[27,54,76,111,171,256],"complexity":[28,150],"$O(H":[30],"\\cdot":[31],"N^2)$":[32],"that":[33,195,240,252],"grows":[34],"quadratically":[35],"with":[36,43,210],"the":[37,44,63,90,110,115,133,180,188,266],"context":[38],"size":[39],"($N$)":[40],"and":[41,190,231,258],"linearly":[42],"number":[45],"heads":[47,58],"($H$).":[48],"This":[49,130],"standard":[50,211],"implementation":[51],"harbors":[52],"significant":[53],"redundancy,":[55],"as":[56,247],"all":[57,234],"independently":[59],"compute":[60],"over":[62],"same":[64],"sequence":[65,182],"space.":[66],"Existing":[67],"sparse":[68,225],"methods,":[69],"meanwhile,":[70],"often":[71],"trade":[72],"information":[73],"integrity":[74],"for":[75,265],"efficiency.":[77],"To":[78],"resolve":[79],"this":[80],"efficiency-performance":[81],"trade-off,":[82],"we":[83,96],"propose":[84],"SPAttention,":[85],"whose":[86],"contribution":[88],"introduction":[91],"new":[94,263],"paradigm":[95],"term":[97],"Principled":[98],"Structural":[99],"Sparsity.":[100],"SPAttention":[101],"does":[102],"not":[103],"merely":[104],"drop":[105],"connections":[106],"but":[107],"instead":[108],"reorganizes":[109],"task":[112],"partitioning":[114],"total":[116],"workload":[118],"into":[119,142],"balanced,":[120],"non-overlapping":[121],"distance":[122],"bands,":[123],"assigning":[124],"each":[125],"head":[126],"unique":[128],"segment.":[129],"approach":[131],"transforms":[132],"multi-head":[134],"mechanism":[136],"from":[137,173],"$H$":[138],"independent":[139],"$O(N^2)$":[140,146],"computations":[141],"single,":[144],"collaborative":[145],"computation,":[147],"fundamentally":[148],"reducing":[149],"factor":[153],"$H$.":[155],"structured":[157],"inductive":[158,250],"bias":[159,251],"compels":[160],"functional":[161],"specialization":[162],"among":[163],"heads,":[164],"enabling":[165],"more":[167],"efficient":[168],"allocation":[169],"resources":[172],"redundant":[174],"modeling":[175],"to":[176],"distinct":[177],"dependencies":[178],"across":[179,233],"entire":[181],"span.":[183],"Extensive":[184],"empirical":[185],"validation":[186],"on":[187,208,217],"OLMoE-1B-7B":[189],"0.25B-1.75B":[191],"model":[192,259],"series":[193],"demonstrates":[194,239],"while":[196,221],"delivering":[197],"an":[198,248],"approximately":[199],"two-fold":[200],"increase":[201],"in":[202],"training":[203],"throughput,":[204],"performance":[206],"par":[209],"dense":[212],"attention,":[213],"even":[214],"surpassing":[215],"it":[216],"select":[218],"key":[219],"metrics,":[220],"consistently":[222],"outperforming":[223],"representative":[224],"methods":[227],"including":[228],"Longformer,":[229],"Reformer,":[230],"BigBird":[232],"evaluation":[235],"metrics.":[236],"Our":[237],"work":[238],"thoughtfully":[241],"designed":[242],"structural":[243],"sparsity":[244],"can":[245],"serve":[246],"effective":[249],"simultaneously":[253],"improves":[254],"both":[255],"efficiency":[257],"performance,":[260],"opening":[261],"avenue":[264],"architectural":[267],"next-generation,":[270],"high-performance":[271],"LLMs.":[272]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-02-13T00:00:00"}