{"id":"https://openalex.org/W7202147293","doi":"https://doi.org/10.48550/arxiv.2608.06849","title":"Autonomy-of-Heads: Data-Free Sparse Attention from Frozen Query-Key Geometry","display_name":"Autonomy-of-Heads: Data-Free Sparse Attention from Frozen Query-Key Geometry","publication_year":2026,"publication_date":"2026-08-07","ids":{"openalex":"https://openalex.org/W7202147293","doi":"https://doi.org/10.48550/arxiv.2608.06849"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.06849","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.06849","pdf_url":"https://arxiv.org/pdf/2608.06849","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.06849","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5146428220","display_name":"Yehan Yang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yang, Yehan","raw_affiliation_strings":["Institute of Computing Technology , Chinese Academy of Sciences","University of Chinese Academy of Sciences"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computing Technology , Chinese Academy of Sciences","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]},{"raw_affiliation_string":"University of Chinese Academy of Sciences","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023311055","display_name":"Junyuan Shang","orcid":"https://orcid.org/0000-0003-4301-750X","observed_orcids":["https://orcid.org/0000-0003-4301-750X"]},"institutions":[{"id":"https://openalex.org/I98301712","display_name":"Baidu (China)","ror":"https://ror.org/03vs3wt56","country_code":"CN","type":"company","lineage":["https://openalex.org/I98301712"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shang, Junyuan","raw_affiliation_strings":["Baidu Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Baidu Inc","institution_ids":["https://openalex.org/I98301712"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5146471554","display_name":"Yang Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li, Yang","raw_affiliation_strings":["Institute of Computing Technology , Chinese Academy of Sciences","University of Chinese Academy of Sciences"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computing Technology , Chinese Academy of Sciences","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]},{"raw_affiliation_string":"University of Chinese Academy of Sciences","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137140997","display_name":"Guanqun Zhao","orcid":"https://orcid.org/0009-0008-5794-202X","observed_orcids":["https://orcid.org/0009-0008-5794-202X"]},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]},{"id":"https://openalex.org/I98301712","display_name":"Baidu (China)","ror":"https://ror.org/03vs3wt56","country_code":"CN","type":"company","lineage":["https://openalex.org/I98301712"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhao, Guanqun","raw_affiliation_strings":["Baidu Inc","Beijing University of Posts and Telecommunications"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Baidu Inc","institution_ids":["https://openalex.org/I98301712"]},{"raw_affiliation_string":"Beijing University of Posts and Telecommunications","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058880150","display_name":"Shuohuan Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I98301712","display_name":"Baidu (China)","ror":"https://ror.org/03vs3wt56","country_code":"CN","type":"company","lineage":["https://openalex.org/I98301712"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wang, Shuohuan","raw_affiliation_strings":["Baidu Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Baidu Inc","institution_ids":["https://openalex.org/I98301712"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5084155236","display_name":"Dianhai Yu","orcid":"https://orcid.org/0000-0002-0163-2603","observed_orcids":["https://orcid.org/0000-0002-0163-2603"]},"institutions":[{"id":"https://openalex.org/I98301712","display_name":"Baidu (China)","ror":"https://ror.org/03vs3wt56","country_code":"CN","type":"company","lineage":["https://openalex.org/I98301712"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu, Dianhai","raw_affiliation_strings":["Baidu Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Baidu Inc","institution_ids":["https://openalex.org/I98301712"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7300000190734863,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7300000190734863,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.12110000103712082,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.08730000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sparse-attention","display_name":"sparse attention","score":0.9929999709129333},{"id":"https://openalex.org/keywords/kv-cache-compression","display_name":"KV cache compression","score":0.5230000019073486},{"id":"https://openalex.org/keywords/long-context-llm-inference","display_name":"long-context LLM inference","score":0.734000027179718},{"id":"https://openalex.org/keywords/effective-rank","display_name":"effective rank","score":0.5680000185966492},{"id":"https://openalex.org/keywords/kv-cache-memory-reduction","display_name":"KV cache memory reduction","score":0.5519999861717224},{"id":"https://openalex.org/keywords/inference-latency","display_name":"inference latency","score":0.6909999847412109}],"concepts":[{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.6614000201225281},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6373000144958496},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.54830002784729},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4900999963283539},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4814999997615814},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.4652000069618225},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.4456000030040741},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.4325000047683716},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.41040000319480896},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.4092999994754791},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.38339999318122864},{"id":"https://openalex.org/C17020691","wikidata":"https://www.wikidata.org/wiki/Q139677","display_name":"Operator (biology)","level":5,"score":0.3790999948978424},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3785000145435333},{"id":"https://openalex.org/C2780312720","wikidata":"https://www.wikidata.org/wiki/Q5689100","display_name":"Head (geology)","level":2,"score":0.3142000138759613},{"id":"https://openalex.org/C92757383","wikidata":"https://www.wikidata.org/wiki/Q382497","display_name":"Affine transformation","level":2,"score":0.2921000123023987},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.28769999742507935},{"id":"https://openalex.org/C152822103","wikidata":"https://www.wikidata.org/wiki/Q7575207","display_name":"Spectral shape analysis","level":3,"score":0.28450000286102295},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.27469998598098755},{"id":"https://openalex.org/C159718280","wikidata":"https://www.wikidata.org/wiki/Q5526353","display_name":"Gas meter prover","level":3,"score":0.27309998869895935},{"id":"https://openalex.org/C2779521785","wikidata":"https://www.wikidata.org/wiki/Q5535529","display_name":"Geometry processing","level":3,"score":0.26499998569488525},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2533000111579895}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.06849","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.06849","pdf_url":"https://arxiv.org/pdf/2608.06849","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.06849","doi":"https://doi.org/10.48550/arxiv.2608.06849","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.06849","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.06849","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.06849","pdf_url":"https://arxiv.org/pdf/2608.06849","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7202147293.pdf","grobid_xml":"https://content.openalex.org/works/W7202147293.grobid-xml"},"referenced_works_count":12,"referenced_works":["https://openalex.org/W1516869117","https://openalex.org/W3015468748","https://openalex.org/W3038287124","https://openalex.org/W4384648639","https://openalex.org/W4386395487","https://openalex.org/W4401306886","https://openalex.org/W4416070557","https://openalex.org/W4417006854","https://openalex.org/W4417530105","https://openalex.org/W7127606931","https://openalex.org/W7127907339","https://openalex.org/W7138293750"],"related_works":["https://openalex.org/W7128490439","https://openalex.org/W4406026730","https://openalex.org/W7138890601","https://openalex.org/W4417530105","https://openalex.org/W7127395297","https://openalex.org/W7162680218","https://openalex.org/W7154151175","https://openalex.org/W7213522280","https://openalex.org/W7223085844","https://openalex.org/W4415798906"],"abstract_inverted_index":{"Long-context":[0],"LLM":[1],"inference":[2],"is":[3],"bottlenecked":[4],"by":[5,162,172],"quadratic":[6],"attention":[7,15,29,70],"computation":[8,127],"and":[9,16,42,56,75,97,115,159,166,169],"growing":[10],"KV-cache":[11,170],"costs.":[12],"Existing":[13],"sparse":[14],"KV-compression":[17],"methods":[18],"typically":[19],"decide":[20],"which":[21],"tokens":[22],"or":[23,35],"heads":[24,58],"to":[25,44,164],"preserve":[26],"from":[27,59],"runtime":[28],"scores,":[30],"observation":[31],"windows,":[32],"calibration":[33],"prompts,":[34],"learned":[36],"gates,":[37],"making":[38],"head":[39,84],"diagnosis":[40],"input-dependent":[41],"costly":[43],"deploy.":[45],"We":[46,121,136],"propose":[47],"Autonomy-of-Heads":[48],"(AoH),":[49],"a":[50,80,89,110],"data-free":[51],"method":[52],"that":[53,128,143],"identifies":[54],"retrieval":[55,101],"streaming":[57,119],"the":[60,68,107,131],"spectral":[61],"geometry":[62],"of":[63,83,92,109,150],"query-key":[64,94],"projections.":[65],"AoH":[66,147],"defines":[67],"kernel":[69],"operator":[71],"$M_h":[72],"=":[73],"W_K^{h\\top}W_Q^h$":[74],"uses":[76],"its":[77],"effective-rank":[78],"as":[79],"weight-space":[81],"measure":[82],"function:":[85],"concentrated":[86],"spectra":[87,105],"indicate":[88,106],"small":[90],"number":[91],"dominant":[93,111],"matching":[95,113],"directions":[96],"are":[98,116],"associated":[99,117],"with":[100,118],"heads,":[102],"whereas":[103],"diffuse":[104],"absence":[108],"global":[112],"direction":[114],"heads.":[120],"further":[122],"derive":[123],"an":[124],"efficient":[125],"$d_\\text{head}$-dimensional":[126],"avoids":[129],"constructing":[130],"full":[132],"$d_\\text{model}\\times":[133],"d_\\text{model}$":[134],"matrix.":[135],"conducted":[137],"extensive":[138],"experiments":[139],"across":[140],"models":[141],"demonstrating":[142],"at":[144,174],"50\\%":[145],"sparsity,":[146],"retains":[148],"96.5\\%":[149],"Full":[151],"Attention":[152],"performance":[153],"on":[154],"average":[155],"while":[156],"reducing":[157],"prefill":[158],"decode":[160],"latency":[161],"up":[163],"41.4\\%":[165],"66.0\\%,":[167],"respectively,":[168],"memory":[171],"50.0\\%":[173],"256K":[175],"tokens.":[176]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-08-11T00:00:00"}