{"id":"https://openalex.org/W4414198630","doi":"https://doi.org/10.1109/tnnls.2025.3606750","title":"AMAP: Automatic Multihead Attention Pruning by Similarity-Based Pruning Indicator","display_name":"AMAP: Automatic Multihead Attention Pruning by Similarity-Based Pruning Indicator","publication_year":2025,"publication_date":"2025-09-15","ids":{"openalex":"https://openalex.org/W4414198630","doi":"https://doi.org/10.1109/tnnls.2025.3606750","pmid":"https://pubmed.ncbi.nlm.nih.gov/40953428"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2025.3606750","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2025.3606750","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016784479","display_name":"Eunho Lee","orcid":null},"institutions":[{"id":"https://openalex.org/I163753206","display_name":"Chungbuk National University","ror":"https://ror.org/02wnxgj78","country_code":"KR","type":"education","lineage":["https://openalex.org/I163753206"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Eunho Lee","raw_affiliation_strings":["Department of Intelligent Systems and Robotics, Chungbuk National University, Cheongju-si, Republic of Korea"],"raw_orcid":"https://orcid.org/0000-0003-1355-7273","affiliations":[{"raw_affiliation_string":"Department of Intelligent Systems and Robotics, Chungbuk National University, Cheongju-si, Republic of Korea","institution_ids":["https://openalex.org/I163753206"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5060901163","display_name":"Youngbae Hwang","orcid":"https://orcid.org/0000-0002-3400-0493"},"institutions":[{"id":"https://openalex.org/I163753206","display_name":"Chungbuk National University","ror":"https://ror.org/02wnxgj78","country_code":"KR","type":"education","lineage":["https://openalex.org/I163753206"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Youngbae Hwang","raw_affiliation_strings":["Department of Intelligent Systems and Robotics, Chungbuk National University, Cheongju-si, Republic of Korea"],"raw_orcid":"https://orcid.org/0000-0002-3400-0493","affiliations":[{"raw_affiliation_string":"Department of Intelligent Systems and Robotics, Chungbuk National University, Cheongju-si, Republic of Korea","institution_ids":["https://openalex.org/I163753206"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I163753206"],"apc_list":null,"apc_paid":null,"fwci":0.4979,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.64048127,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":"37","issue":"1","first_page":"357","last_page":"370"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.9508000016212463,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.9508000016212463,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.8906000256538391},{"id":"https://openalex.org/keywords/flops","display_name":"FLOPS","score":0.7770000100135803},{"id":"https://openalex.org/keywords/initialization","display_name":"Initialization","score":0.7451000213623047},{"id":"https://openalex.org/keywords/reduction","display_name":"Reduction (mathematics)","score":0.5697000026702881},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.5406000018119812},{"id":"https://openalex.org/keywords/channel","display_name":"Channel (broadcasting)","score":0.48969998955726624},{"id":"https://openalex.org/keywords/lossless-compression","display_name":"Lossless compression","score":0.43790000677108765}],"concepts":[{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.8906000256538391},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7997000217437744},{"id":"https://openalex.org/C3826847","wikidata":"https://www.wikidata.org/wiki/Q188768","display_name":"FLOPS","level":2,"score":0.7770000100135803},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.7451000213623047},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.5697000026702881},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.5406000018119812},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5223000049591064},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.48969998955726624},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.47040000557899475},{"id":"https://openalex.org/C81081738","wikidata":"https://www.wikidata.org/wiki/Q55542","display_name":"Lossless compression","level":3,"score":0.43790000677108765},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.40799999237060547},{"id":"https://openalex.org/C179799912","wikidata":"https://www.wikidata.org/wiki/Q205084","display_name":"Computational complexity theory","level":2,"score":0.4027999937534332},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.39959999918937683},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3939000070095062},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.34450000524520874},{"id":"https://openalex.org/C311688","wikidata":"https://www.wikidata.org/wiki/Q2393193","display_name":"Time complexity","level":2,"score":0.29739999771118164},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2782999873161316},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.26510000228881836}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2025.3606750","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2025.3606750","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:40953428","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40953428","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2542981700","display_name":null,"funder_award_id":"RS-2024-00425661","funder_id":"https://openalex.org/F4320335489","funder_display_name":"Institute for Information and Communications Technology Promotion"},{"id":"https://openalex.org/G490925689","display_name":null,"funder_award_id":"RS\u20102021\u2010II212068","funder_id":"https://openalex.org/F4320335489","funder_display_name":"Institute for Information and Communications Technology Promotion"}],"funders":[{"id":"https://openalex.org/F4320335489","display_name":"Institute for Information and Communications Technology Promotion","ror":"https://ror.org/01g0hqq23"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":40,"referenced_works":["https://openalex.org/W2108598243","https://openalex.org/W2752782242","https://openalex.org/W2984618279","https://openalex.org/W3034513523","https://openalex.org/W3107407793","https://openalex.org/W3121523901","https://openalex.org/W3131500599","https://openalex.org/W3138516171","https://openalex.org/W3202742610","https://openalex.org/W4200630931","https://openalex.org/W4214493665","https://openalex.org/W4214634256","https://openalex.org/W4283811336","https://openalex.org/W4312796067","https://openalex.org/W4312950730","https://openalex.org/W4312960790","https://openalex.org/W4320036918","https://openalex.org/W4323644242","https://openalex.org/W4366241503","https://openalex.org/W4377710209","https://openalex.org/W4383899306","https://openalex.org/W4385245566","https://openalex.org/W4386065441","https://openalex.org/W4386066311","https://openalex.org/W4386072014","https://openalex.org/W4386075553","https://openalex.org/W4386076285","https://openalex.org/W4386076556","https://openalex.org/W4387587615","https://openalex.org/W4390871883","https://openalex.org/W4390872447","https://openalex.org/W4390873032","https://openalex.org/W4390874124","https://openalex.org/W4391021637","https://openalex.org/W4392940376","https://openalex.org/W4394625683","https://openalex.org/W4402215966","https://openalex.org/W4402816727","https://openalex.org/W4404851820","https://openalex.org/W4405934961"],"related_works":[],"abstract_inverted_index":{"Despite":[0],"the":[1,107,112,122,165,169,181,212,225,245],"strong":[2,30],"performance":[3,49],"of":[4,9,197,208],"transformers,":[5],"quadratic":[6,25],"computation":[7],"complexity":[8,23],"self-attention":[10],"presents":[11],"challenges":[12],"in":[13,199,232],"applying":[14,62,176],"them":[15],"to":[16,26,43,64,71,83,110,125,145,150,180],"vision":[17],"tasks.":[18],"Linear":[19],"attention":[20,66,166,190],"reduces":[21],"this":[22,75,86],"from":[24,89],"linear,":[27],"offering":[28],"a":[29,45,51,142,194,200,229],"computation-performance":[31],"tradeoff.":[32],"To":[33],"further":[34],"optimize":[35],"this,":[36],"automatic":[37,80],"pruning":[38,81,108,123,157,178,247],"is":[39,67],"an":[40,79,155],"effective":[41,156],"method":[42,82,179,238],"find":[44],"structure":[46,171],"that":[47,92,127],"maximizes":[48],"within":[50,116],"target":[52],"resource":[53],"through":[54],"training":[55,96],"without":[56,97],"any":[57,98,137],"heuristic":[58],"approaches.":[59],"However,":[60],"directly":[61],"it":[63],"multihead":[65],"not":[68],"straightforward":[69],"due":[70,149],"channel":[72,103,138,151],"mismatch.":[73,139],"In":[74],"article,":[76],"we":[77,101,120,192],"propose":[78],"deal":[84],"with":[85,228],"problem.":[87],"Different":[88],"existing":[90],"methods":[91],"rely":[93],"solely":[94],"on":[95,164,183],"prior":[99],"knowledge,":[100],"integrate":[102],"similarity-based":[104],"weights":[105],"into":[106],"indicator":[109,124,158],"preserve":[111],"more":[113],"informative":[114],"channels":[115,128],"each":[117,173],"head.":[118],"Then,":[119],"adjust":[121],"enforce":[126],"are":[129],"removed":[130],"evenly":[131],"across":[132],"all":[133],"heads,":[134],"thereby":[135],"avoiding":[136],"We":[140],"incorporate":[141],"reweight":[143],"module":[144],"mitigate":[146],"information":[147],"loss":[148],"removal":[152],"and":[153,172,188,220,244],"introduce":[154],"initialization":[159],"for":[160],"linear":[161,189],"attention,":[162],"based":[163],"differences":[167],"between":[168],"original":[170,187],"channel.":[174],"By":[175],"our":[177],"FLattenTransformer":[182],"ImageNet-1K,":[184],"which":[185],"incorporates":[186],"mechanisms,":[191],"achieve":[193],"30%":[195],"reduction":[196,231],"FLOPs":[198,217,233],"near":[201],"lossless":[202],"manner.":[203],"It":[204],"also":[205],"has":[206],"1.96%":[207],"accuracy":[209,222],"gain":[210],"over":[211,224],"DeiT-B":[213],"model":[214,227],"while":[215],"reducing":[216],"by":[218],"37%,":[219],"1.05%":[221],"increase":[223],"Swin-B":[226],"10%":[230],"as":[234],"well.":[235],"The":[236],"proposed":[237],"outperforms":[239],"previous":[240],"state-of-the-art":[241],"efficient":[242],"models":[243],"recent":[246],"methods.":[248]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-08-01T09:00:35.917206","created_date":"2025-10-10T00:00:00"}