{"id":"https://openalex.org/W7161304091","doi":"https://doi.org/10.1016/j.neucom.2026.133976","title":"Rethinking attention reliability for token pruning in vision transformers","display_name":"Rethinking attention reliability for token pruning in vision transformers","publication_year":2026,"publication_date":"2026-05-15","ids":{"openalex":"https://openalex.org/W7161304091","doi":"https://doi.org/10.1016/j.neucom.2026.133976"},"language":"en","primary_location":{"id":"doi:10.1016/j.neucom.2026.133976","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.neucom.2026.133976","pdf_url":null,"source":{"id":"https://openalex.org/S45693802","display_name":"Neurocomputing","issn_l":"0925-2312","issn":["0925-2312","1872-8286"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neurocomputing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1016/j.neucom.2026.133976","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053005436","display_name":"Ryuto Ishibashi","orcid":"https://orcid.org/0009-0000-3161-2200"},"institutions":[{"id":"https://openalex.org/I135768898","display_name":"Ritsumeikan University","ror":"https://ror.org/0197nmd03","country_code":"JP","type":"education","lineage":["https://openalex.org/I135768898","https://openalex.org/I4390039241"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Ryuto Ishibashi","raw_affiliation_strings":["Graduate School of Science and Engineering, Ritsumeikan University, 1-1-1 Noji-higashi, Kusatsu, 525-8577, Shiga, Japan"],"raw_orcid":"https://orcid.org/0009-0000-3161-2200","affiliations":[{"raw_affiliation_string":"Graduate School of Science and Engineering, Ritsumeikan University, 1-1-1 Noji-higashi, Kusatsu, 525-8577, Shiga, Japan","institution_ids":["https://openalex.org/I135768898"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136265189","display_name":"Hayata Kaneko","orcid":"https://orcid.org/0009-0003-2294-5208"},"institutions":[{"id":"https://openalex.org/I135768898","display_name":"Ritsumeikan University","ror":"https://ror.org/0197nmd03","country_code":"JP","type":"education","lineage":["https://openalex.org/I135768898","https://openalex.org/I4390039241"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Hayata Kaneko","raw_affiliation_strings":["Graduate School of Science and Engineering, Ritsumeikan University, 1-1-1 Noji-higashi, Kusatsu, 525-8577, Shiga, Japan"],"raw_orcid":"https://orcid.org/0009-0003-2294-5208","affiliations":[{"raw_affiliation_string":"Graduate School of Science and Engineering, Ritsumeikan University, 1-1-1 Noji-higashi, Kusatsu, 525-8577, Shiga, Japan","institution_ids":["https://openalex.org/I135768898"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5136212238","display_name":"Lin Meng","orcid":"https://orcid.org/0000-0003-4351-6923"},"institutions":[{"id":"https://openalex.org/I135768898","display_name":"Ritsumeikan University","ror":"https://ror.org/0197nmd03","country_code":"JP","type":"education","lineage":["https://openalex.org/I135768898","https://openalex.org/I4390039241"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Lin Meng","raw_affiliation_strings":["College of Science and Engineering, Ritsumeikan University, 1-1-1 Noji-higashi, Kusatsu, 525-8577, Shiga, Japan"],"raw_orcid":"https://orcid.org/0000-0003-4351-6923","affiliations":[{"raw_affiliation_string":"College of Science and Engineering, Ritsumeikan University, 1-1-1 Noji-higashi, Kusatsu, 525-8577, Shiga, Japan","institution_ids":["https://openalex.org/I135768898"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5136212238"],"corresponding_institution_ids":["https://openalex.org/I135768898"],"apc_list":{"value":2670,"currency":"USD","value_usd":2670},"apc_paid":{"value":2670,"currency":"USD","value_usd":2670},"fwci":4.7465,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.95377508,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":"694","issue":null,"first_page":"133976","last_page":"133976"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.4397999942302704,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.4397999942302704,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11992","display_name":"CCD and CMOS Imaging Sensors","score":0.15950000286102295,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13418","display_name":"Photocathodes and Microchannel Plates","score":0.05640000104904175,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6747000217437744},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.6001999974250793},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5920000076293945},{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.43689998984336853}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7677000164985657},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6747000217437744},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6001999974250793},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5990999937057495},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5920000076293945},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46790000796318054},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.43689998984336853},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.37279999256134033},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.2590000033378601},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.25589999556541443}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1016/j.neucom.2026.133976","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.neucom.2026.133976","pdf_url":null,"source":{"id":"https://openalex.org/S45693802","display_name":"Neurocomputing","issn_l":"0925-2312","issn":["0925-2312","1872-8286"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neurocomputing","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1016/j.neucom.2026.133976","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.neucom.2026.133976","pdf_url":null,"source":{"id":"https://openalex.org/S45693802","display_name":"Neurocomputing","issn_l":"0925-2312","issn":["0925-2312","1872-8286"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neurocomputing","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G760205660","display_name":null,"funder_award_id":"JPMJBS2428","funder_id":"https://openalex.org/F4320334789","funder_display_name":"Japan Science and Technology Agency"}],"funders":[{"id":"https://openalex.org/F4320320907","display_name":"Japan Science and Technology Corporation","ror":"https://ror.org/00097mb19"},{"id":"https://openalex.org/F4320334789","display_name":"Japan Science and Technology Agency","ror":"https://ror.org/00097mb19"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":2,"referenced_works":["https://openalex.org/W2078483536","https://openalex.org/W4406505715"],"related_works":[],"abstract_inverted_index":{"Vision":[0],"Transformers":[1],"(ViTs)":[2],"achieve":[3,245],"strong":[4],"performance":[5],"in":[6,95,119,163,272],"visual":[7],"recognition":[8],"but":[9],"incur":[10],"quadratic":[11],"computational":[12],"cost":[13],"with":[14,66],"respect":[15],"to":[16,99,200,255,258],"the":[17,59,77,179,201,221,241],"number":[18],"of":[19,61,84,133,204,223,260],"tokens,":[20],"motivating":[21],"extensive":[22],"research":[23],"on":[24,76,207],"token":[25,34,123,139,210,262,269],"pruning":[26,31,161,196,211,243],"and":[27,69,72,90,103,143,151,188,225,250,285],"reduction.":[28],"Most":[29],"existing":[30],"methods":[32],"estimate":[33],"importance":[35,135,270],"directly":[36],"from":[37,138],"attention":[38,43,85,93,115,291],"weights,":[39],"implicitly":[40],"assuming":[41],"that":[42,58,193,252,259],"magnitude":[44],"provides":[45],"a":[46,81,130,215,246],"reliable":[47],"proxy":[48],"for":[49,110],"semantic":[50],"relevance":[51],"across":[52],"all":[53],"layers.":[54],"Our":[55],"analysis":[56,83],"shows":[57],"validity":[60],"this":[62,128,183],"assumption":[63],"varies":[64],"substantially":[65],"transformer":[67],"depth":[68,180],"model":[70,283],"scale,":[71],"can":[73],"also":[74],"depend":[75],"training":[78,153,286],"paradigm.":[79],"Through":[80],"systematic":[82],"selectivity":[86],"using":[87],"multiple":[88],"concentration":[89],"stability":[91],"measures,":[92],"distributions":[94],"shallow":[96,164],"layers":[97,121],"tend":[98],"be":[100],"highly":[101],"diffuse":[102],"weakly":[104],"discriminative,":[105],"making":[106],"attention-based":[107,167,195],"scoring":[108,168],"unreliable":[109],"early-stage":[111],"pruning.":[112],"In":[113],"contrast,":[114],"becomes":[116,169],"increasingly":[117],"informative":[118],"deeper":[120],"as":[122,214,274],"representations":[124],"mature.":[125],"Motivated":[126],"by":[127,220,280],"observation,":[129],"broad":[131],"range":[132],"non-attention":[134,156],"scores":[136,157],"derived":[137],"embeddings,":[140],"including":[141],"statistics-":[142],"similarity-based":[144],"criteria,":[145],"is":[146,176,186,197,212,253],"examined.":[147],"Across":[148],"ViT":[149],"variants":[150],"diverse":[152],"settings,":[154],"these":[155,208,266],"exhibit":[158],"more":[159],"stable":[160],"behavior":[162],"layers,":[165],"whereas":[166],"effective":[170],"only":[171],"after":[172],"sufficient":[173],"representational":[174,202],"discrimination":[175],"achieved.":[177],"Importantly,":[178],"at":[181],"which":[182],"transition":[184],"occurs":[185],"model-dependent":[187],"not":[189],"strictly":[190],"monotonic,":[191],"indicating":[192],"uniform":[194,290],"fundamentally":[198],"mismatched":[199],"dynamics":[203],"ViTs.":[205],"Based":[206],"findings,":[209],"formulated":[213],"layer-wise":[216],"selection":[217],"problem":[218,278],"governed":[219],"reliability":[222],"attention,":[224],"lightweight":[226],"static":[227],"routing":[228],"configurations":[229],"are":[230],"investigated":[231],"without":[232],"retraining":[233],"or":[234,256],"dynamic":[235],"inference":[236],"control.":[237],"For":[238],"equivalent":[239],"FLOPs,":[240],"resulting":[242],"patterns":[244],"trade-off":[247],"between":[248],"accuracy":[249],"efficiency":[251],"comparable":[254],"superior":[257],"representative":[261],"reduction":[263],"methods.":[264],"Overall,":[265],"results":[267],"establish":[268],"estimation":[271],"ViTs":[273],"an":[275],"inherently":[276],"layer-dependent":[277],"shaped":[279],"representation":[281],"maturity,":[282],"characteristics,":[284],"paradigm":[287],"rather":[288],"than":[289],"magnitude.":[292]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-08-29T07:29:34.045763","created_date":"2026-05-16T00:00:00"}