{"id":"https://openalex.org/W4385488555","doi":"https://doi.org/10.1109/ijcnn54540.2023.10191462","title":"Dense Attention: A Densely Connected Attention Mechanism for Vision Transformer","display_name":"Dense Attention: A Densely Connected Attention Mechanism for Vision Transformer","publication_year":2023,"publication_date":"2023-06-18","ids":{"openalex":"https://openalex.org/W4385488555","doi":"https://doi.org/10.1109/ijcnn54540.2023.10191462"},"language":"en","primary_location":{"id":"doi:10.1109/ijcnn54540.2023.10191462","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn54540.2023.10191462","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100419518","display_name":"Nannan Li","orcid":"https://orcid.org/0000-0001-5563-3735"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Nannan Li","raw_affiliation_strings":["Institute of Automation,State Key Laboratory of Multimodal Artificial Intelligence Systems, Chinese Academy of Sciences,Beijing,China,100190","School of artificial intelligence, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Automation,State Key Laboratory of Multimodal Artificial Intelligence Systems, Chinese Academy of Sciences,Beijing,China,100190","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"raw_affiliation_string":"School of artificial intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053853035","display_name":"Yaran Chen","orcid":"https://orcid.org/0000-0001-9356-0610"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yaran Chen","raw_affiliation_strings":["Institute of Automation,State Key Laboratory of Multimodal Artificial Intelligence Systems, Chinese Academy of Sciences,Beijing,China,100190","School of artificial intelligence, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Automation,State Key Laboratory of Multimodal Artificial Intelligence Systems, Chinese Academy of Sciences,Beijing,China,100190","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"raw_affiliation_string":"School of artificial intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4210165038"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100624298","display_name":"Dongbin Zhao","orcid":"https://orcid.org/0000-0001-8218-9633"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongbin Zhao","raw_affiliation_strings":["Institute of Automation,State Key Laboratory of Multimodal Artificial Intelligence Systems, Chinese Academy of Sciences,Beijing,China,100190","School of artificial intelligence, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Automation,State Key Laboratory of Multimodal Artificial Intelligence Systems, Chinese Academy of Sciences,Beijing,China,100190","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"raw_affiliation_string":"School of artificial intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4210165038"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7107206583023071},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6003736853599548},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5693551301956177},{"id":"https://openalex.org/keywords/architecture","display_name":"Architecture","score":0.5277907252311707},{"id":"https://openalex.org/keywords/soar","display_name":"Soar","score":0.526157557964325},{"id":"https://openalex.org/keywords/transferability","display_name":"Transferability","score":0.48632723093032837},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.34917551279067993}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7107206583023071},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6003736853599548},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5693551301956177},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.5277907252311707},{"id":"https://openalex.org/C17305859","wikidata":"https://www.wikidata.org/wiki/Q382944","display_name":"Soar","level":2,"score":0.526157557964325},{"id":"https://openalex.org/C61272859","wikidata":"https://www.wikidata.org/wiki/Q7834031","display_name":"Transferability","level":3,"score":0.48632723093032837},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34917551279067993},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C140331021","wikidata":"https://www.wikidata.org/wiki/Q1868104","display_name":"Logit","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ijcnn54540.2023.10191462","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn54540.2023.10191462","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.47999998927116394,"display_name":"Industry, innovation and infrastructure"}],"awards":[{"id":"https://openalex.org/G644626782","display_name":null,"funder_award_id":"62173324","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6882043578","display_name":null,"funder_award_id":"104GJHZ2022013GC","funder_id":"https://openalex.org/F4320321133","funder_display_name":"Chinese Academy of Sciences"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321133","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":46,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1977295820","https://openalex.org/W2108598243","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2470673105","https://openalex.org/W2524771588","https://openalex.org/W2732016772","https://openalex.org/W2899663614","https://openalex.org/W2955425717","https://openalex.org/W2962843773","https://openalex.org/W2962988160","https://openalex.org/W2963446712","https://openalex.org/W2982083293","https://openalex.org/W3034429256","https://openalex.org/W3035422918","https://openalex.org/W3035452548","https://openalex.org/W3038678879","https://openalex.org/W3094502228","https://openalex.org/W3131500599","https://openalex.org/W3138516171","https://openalex.org/W3138562265","https://openalex.org/W3139445856","https://openalex.org/W3163465952","https://openalex.org/W3170874841","https://openalex.org/W3172509117","https://openalex.org/W3198221608","https://openalex.org/W3204801262","https://openalex.org/W4214493665","https://openalex.org/W4214588794","https://openalex.org/W4226410605","https://openalex.org/W4367663369","https://openalex.org/W4376226279","https://openalex.org/W4385245566","https://openalex.org/W6631190155","https://openalex.org/W6685627644","https://openalex.org/W6739901393","https://openalex.org/W6755977528","https://openalex.org/W6762718338","https://openalex.org/W6783246024","https://openalex.org/W6784333009","https://openalex.org/W6788135285","https://openalex.org/W6796417832","https://openalex.org/W6801443639","https://openalex.org/W6811407177","https://openalex.org/W6839710751"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W4224009465","https://openalex.org/W4286629047","https://openalex.org/W4306321456","https://openalex.org/W4285260836","https://openalex.org/W3046775127","https://openalex.org/W3170094116","https://openalex.org/W4205958290","https://openalex.org/W4386462264"],"abstract_inverted_index":{"Recently,":[0],"Vision":[1,119],"Transformer":[2,120],"has":[3,23],"demonstrated":[4],"its":[5,18,162],"impressive":[6],"capability":[7],"in":[8,116,190],"image":[9,89],"understanding.":[10],"The":[11,131],"multi-head":[12,99],"self-attention":[13,22,100],"mechanism":[14,52,101],"is":[15],"fundamental":[16],"to":[17,55,63,102,143],"formidable":[19],"performance.":[20],"However,":[21],"the":[24,32,35,57,77,88,92,98,108,117,183,188,191],"drawback":[25],"of":[26,34,79,91,110,173,193],"high":[27],"computational":[28,39],"effort,":[29],"which":[30],"makes":[31],"training":[33],"model":[36,104],"require":[37],"powerful":[38],"resources":[40],"or":[41],"more":[42],"time.":[43],"This":[44],"paper":[45],"designs":[46],"a":[47,71],"novel":[48],"and":[49,94,126,151,166,175,196],"efficient":[50],"attention":[51,61,78,83,140,159,180],"Dense":[53,60,158],"Attention":[54],"overcome":[56],"above":[58],"problem.":[59],"aims":[62],"focus":[64],"on":[65,134,164],"features":[66],"from":[67,76],"multiple":[68],"views":[69],"through":[70],"dense":[72,82,111,139,179],"connection":[73],"paradigm.":[74],"Benefiting":[75],"comprehensive":[80],"features,":[81],"can":[84,181],"i)":[85],"remarkably":[86],"strengthen":[87],"representation":[90],"model,":[93],"ii)":[95],"partially":[96],"replace":[97],"allow":[103],"slimming.":[105],"To":[106],"verify":[107],"effectiveness":[109],"attention,":[112],"we":[113],"implement":[114],"it":[115],"prevalent":[118],"models,":[121],"including":[122],"non-pyramid":[123],"architecture":[124,128],"DeiT":[125],"pyramid":[127],"Swin":[129],"Transformer.":[130],"experimental":[132],"results":[133],"ImageNet":[135],"classification":[136,171],"show":[137],"that":[138],"indeed":[141],"contributes":[142],"performance":[144,184],"improvement,":[145],"<tex":[146,152],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[147,153,203,206],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$+\\mathbf{1.8}/\\mathbf{1.3}\\%$</tex>":[148],"for":[149,155],"DeiT-T/s":[150],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$+\\mathbf{0.7}/\\!\\!+\\mathbf{1.2}\\%$</tex>":[154],"Swin-T/s,":[156],"respectively.":[157,177],"also":[160],"demonstrates":[161],"transferability":[163],"CIFAR10":[165],"CIFAR100":[167],"recognition":[168],"benchmarks":[169],"with":[170],"accuracy":[172],"98.9%":[174],"89.6%":[176],"Furthermore,":[178],"weaken":[182],"sacrifice":[185],"caused":[186],"by":[187],"pruning":[189],"number":[192],"heads.":[194],"Code":[195],"pre-trained":[197],"models":[198],"will":[199],"be":[200],"available":[201],"<sup":[202,205],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">1</sup>":[204,207],"https://github.com/koala719/Dense-ViT.":[208]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}