{"id":"https://openalex.org/W4415428212","doi":"https://doi.org/10.3233/faia251302","title":"Investigating Sparsity of Self-Attention","display_name":"Investigating Sparsity of Self-Attention","publication_year":2025,"publication_date":"2025-10-21","ids":{"openalex":"https://openalex.org/W4415428212","doi":"https://doi.org/10.3233/faia251302"},"language":null,"primary_location":{"id":"doi:10.3233/faia251302","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia251302","pdf_url":null,"source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"},"type":"book-chapter","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.3233/faia251302","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120092174","display_name":"Martin Garaj","orcid":null},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Martin Garaj","raw_affiliation_strings":["Huawei, Hong Kong Research Center, Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei, Hong Kong Research Center, Hong Kong","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074029092","display_name":"Kisub Kim","orcid":"https://orcid.org/0000-0002-4462-6916"},"institutions":[{"id":"https://openalex.org/I193352282","display_name":"Daegu Gyeongbuk Institute of Science and Technology","ror":"https://ror.org/03frjya69","country_code":"KR","type":"education","lineage":["https://openalex.org/I193352282"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Kisub Kim","raw_affiliation_strings":["DGIST, Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"DGIST, Korea","institution_ids":["https://openalex.org/I193352282"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5120092175","display_name":"Alexis Stockinger","orcid":null},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Alexis Stockinger","raw_affiliation_strings":["Huawei, Hong Kong Research Center, Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei, Hong Kong Research Center, Hong Kong","institution_ids":["https://openalex.org/I2250955327"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.59124311,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13219","display_name":"Mind wandering and attention","score":0.9193999767303467,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T13219","display_name":"Mind wandering and attention","score":0.9193999767303467,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.6876999735832214},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5957000255584717},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.5807999968528748},{"id":"https://openalex.org/keywords/mechanism","display_name":"Mechanism (biology)","score":0.5515000224113464},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4810999929904938},{"id":"https://openalex.org/keywords/cluster-analysis","display_name":"Cluster analysis","score":0.4348999857902527},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.35100001096725464}],"concepts":[{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.6876999735832214},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6680999994277954},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5957000255584717},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5807999968528748},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.5515000224113464},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4810999929904938},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47440001368522644},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.4348999857902527},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3684000074863434},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.35100001096725464},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3248000144958496},{"id":"https://openalex.org/C167927819","wikidata":"https://www.wikidata.org/wiki/Q1930567","display_name":"Shuffling","level":2,"score":0.3246999979019165},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.3041999936103821},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.2948000133037567},{"id":"https://openalex.org/C32946077","wikidata":"https://www.wikidata.org/wiki/Q618079","display_name":"Network analysis","level":2,"score":0.2928999960422516},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.260699987411499},{"id":"https://openalex.org/C72169020","wikidata":"https://www.wikidata.org/wiki/Q194404","display_name":"Monotonic function","level":2,"score":0.2581999897956848},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.25459998846054077}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.3233/faia251302","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia251302","pdf_url":null,"source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"}],"best_oa_location":{"id":"doi:10.3233/faia251302","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia251302","pdf_url":null,"source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Understanding":[0],"the":[1,5,72,130],"sparsity":[2,30,45],"patterns":[3,183],"of":[4,132],"self-attention":[6,44],"mechanism":[7,200],"in":[8,48,162,184],"modern":[9],"Large":[10],"Language":[11],"Models":[12],"(LLMs)":[13],"has":[14],"become":[15],"increasingly":[16],"important":[17,81,142],"for":[18,125,196],"improving":[19],"computational":[20],"efficiency.":[21],"Motivated":[22],"by":[23],"empirical":[24],"observations,":[25],"numerous":[26],"algorithms":[27,178],"assume":[28],"specific":[29],"structures":[31],"within":[32],"self-attention.":[33],"In":[34],"this":[35],"work,":[36],"we":[37],"rigorously":[38],"examine":[39],"five":[40],"common":[41],"conjectures":[42,111],"about":[43],"frequently":[46],"addressed":[47],"recent":[49,66,113],"literature:":[50],"(1)":[51],"attention":[52,59,92,115,145,159,177,182,199],"width":[53,146,160],"decreases":[54],"through":[55],"network":[56],"depth,":[57],"(2)":[58,128],"heads":[60],"form":[61],"distinct":[62],"behavioral":[63],"clusters,":[64],"(3)":[65,116],"tokens":[67,82],"receive":[68],"high":[69],"attention,":[70],"(4)":[71],"first":[73],"token":[74,114],"maintains":[75],"consistent":[76],"focus,":[77],"and":[78,117,129],"(5)":[79],"semantically":[80],"persistently":[83],"attract":[84],"attention.":[85],"Our":[86,107],"analysis":[87,149],"uses":[88],"over":[89,98],"4":[90],"million":[91],"weight":[93],"vectors":[94],"from":[95],"Llama3-8B":[96],"collected":[97],"long-context":[99],"benchmark":[100],"LongBench":[101],"to":[102],"achieve":[103],"statistically":[104],"significant":[105],"results.":[106],"findings":[108],"strongly":[109],"support":[110,124],"regarding":[112],"first-token":[118],"focus":[119],"(4).":[120],"We":[121],"find":[122],"partial":[123],"head":[126],"clustering":[127],"Persistence":[131],"Attention":[133],"Hypothesis":[134],"(5),":[135],"suggesting":[136],"these":[137],"phenomena":[138],"exist":[139],"but":[140],"with":[141,158,169],"qualifications.":[143],"Regarding":[144],"(1),":[147],"our":[148],"reveals":[150],"a":[151],"more":[152,189,197],"nuanced":[153],"pattern":[154],"than":[155,166],"commonly":[156],"assumed,":[157],"peaking":[161],"middle":[163,185],"layers":[164,186],"rather":[165],"decreasing":[167],"monotonically":[168],"depth.":[170],"These":[171],"insights":[172],"suggest":[173],"that":[174],"effective":[175],"sparse":[176],"should":[179],"preserve":[180],"broader":[181],"while":[187],"allowing":[188],"targeted":[190],"pruning":[191],"elsewhere,":[192],"offering":[193],"evidence-based":[194],"guidance":[195],"efficient":[198],"design.":[201]},"counts_by_year":[],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-24T00:00:00"}