{"id":"https://openalex.org/W4362500802","doi":"https://doi.org/10.1109/tpami.2023.3263826","title":"Dynamic Spatial Sparsification for Efficient Vision Transformers and Convolutional Neural Networks","display_name":"Dynamic Spatial Sparsification for Efficient Vision Transformers and Convolutional Neural Networks","publication_year":2023,"publication_date":"2023-04-03","ids":{"openalex":"https://openalex.org/W4362500802","doi":"https://doi.org/10.1109/tpami.2023.3263826","pmid":"https://pubmed.ncbi.nlm.nih.gov/37030709"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2023.3263826","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2023.3263826","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5082694832","display_name":"Yongming Rao","orcid":"https://orcid.org/0000-0003-3952-8753"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yongming Rao","raw_affiliation_strings":["Department of Automation, Beijing National Research Center for Information Science and Technology (BNRist), Tsinghua University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Department of Automation, Beijing National Research Center for Information Science and Technology (BNRist), Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013372143","display_name":"Zuyan Liu","orcid":"https://orcid.org/0009-0002-6943-3085"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zuyan Liu","raw_affiliation_strings":["Department of Automation, Beijing National Research Center for Information Science and Technology (BNRist), Tsinghua University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Department of Automation, Beijing National Research Center for Information Science and Technology (BNRist), Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060902476","display_name":"Wenliang Zhao","orcid":"https://orcid.org/0000-0002-0920-1576"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenliang Zhao","raw_affiliation_strings":["Department of Automation, Beijing National Research Center for Information Science and Technology (BNRist), Tsinghua University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Department of Automation, Beijing National Research Center for Information Science and Technology (BNRist), Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100620306","display_name":"Jie Zhou","orcid":"https://orcid.org/0000-0001-7701-234X"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jie Zhou","raw_affiliation_strings":["Department of Automation, Beijing National Research Center for Information Science and Technology (BNRist), Tsinghua University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Department of Automation, Beijing National Research Center for Information Science and Technology (BNRist), Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100460385","display_name":"Jiwen Lu","orcid":"https://orcid.org/0000-0002-6121-5529"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiwen Lu","raw_affiliation_strings":["Department of Automation, Beijing National Research Center for Information Science and Technology (BNRist), Tsinghua University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Department of Automation, Beijing National Research Center for Information Science and Technology (BNRist), Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5082694832"],"corresponding_institution_ids":["https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":6.7402,"has_fulltext":false,"cited_by_count":56,"citation_normalized_percentile":{"value":0.97829271,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"45","issue":"9","first_page":"10883","last_page":"10897"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8177037239074707},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.7148559093475342},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5988204479217529},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5871078372001648},{"id":"https://openalex.org/keywords/flops","display_name":"FLOPS","score":0.5821785926818848},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.5347565412521362},{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.49449869990348816},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.4126584529876709},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.39591604471206665},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.27753016352653503},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.10244804620742798}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8177037239074707},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.7148559093475342},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5988204479217529},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5871078372001648},{"id":"https://openalex.org/C3826847","wikidata":"https://www.wikidata.org/wiki/Q188768","display_name":"FLOPS","level":2,"score":0.5821785926818848},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5347565412521362},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.49449869990348816},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4126584529876709},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.39591604471206665},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.27753016352653503},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.10244804620742798},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2023.3263826","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2023.3263826","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:37030709","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37030709","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3510215249","display_name":null,"funder_award_id":"62125603","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":90,"referenced_works":["https://openalex.org/W1724438581","https://openalex.org/W1821462560","https://openalex.org/W1861492603","https://openalex.org/W2086161653","https://openalex.org/W2108598243","https://openalex.org/W2163605009","https://openalex.org/W2194775991","https://openalex.org/W2624273542","https://openalex.org/W2737258237","https://openalex.org/W2754084392","https://openalex.org/W2889469641","https://openalex.org/W2896409484","https://openalex.org/W2898170443","https://openalex.org/W2905741102","https://openalex.org/W2910628332","https://openalex.org/W2915716523","https://openalex.org/W2962858109","https://openalex.org/W2962935523","https://openalex.org/W2963125977","https://openalex.org/W2963150697","https://openalex.org/W2963163009","https://openalex.org/W2963363373","https://openalex.org/W2963393494","https://openalex.org/W2963393566","https://openalex.org/W2963526497","https://openalex.org/W2963910742","https://openalex.org/W2967733054","https://openalex.org/W2982479999","https://openalex.org/W2990503944","https://openalex.org/W2990631821","https://openalex.org/W3034429256","https://openalex.org/W3035071066","https://openalex.org/W3035678286","https://openalex.org/W3094502228","https://openalex.org/W3096609285","https://openalex.org/W3101720316","https://openalex.org/W3105966348","https://openalex.org/W3107016329","https://openalex.org/W3121523901","https://openalex.org/W3128633047","https://openalex.org/W3131500599","https://openalex.org/W3133696297","https://openalex.org/W3137963805","https://openalex.org/W3138516171","https://openalex.org/W3139049060","https://openalex.org/W3151130473","https://openalex.org/W3153842237","https://openalex.org/W3170841864","https://openalex.org/W3176196997","https://openalex.org/W3188427387","https://openalex.org/W3203898101","https://openalex.org/W3204801262","https://openalex.org/W4214493665","https://openalex.org/W4214633470","https://openalex.org/W4214709605","https://openalex.org/W4214713996","https://openalex.org/W4214755140","https://openalex.org/W4221156361","https://openalex.org/W4297775537","https://openalex.org/W4312290555","https://openalex.org/W4312340826","https://openalex.org/W4312443924","https://openalex.org/W4312849330","https://openalex.org/W4312872526","https://openalex.org/W4322716158","https://openalex.org/W4385245566","https://openalex.org/W6637709462","https://openalex.org/W6638523607","https://openalex.org/W6684191040","https://openalex.org/W6729448088","https://openalex.org/W6737664043","https://openalex.org/W6739663239","https://openalex.org/W6739901393","https://openalex.org/W6743912273","https://openalex.org/W6755843862","https://openalex.org/W6757036269","https://openalex.org/W6762718338","https://openalex.org/W6784233108","https://openalex.org/W6784333009","https://openalex.org/W6788135285","https://openalex.org/W6790428460","https://openalex.org/W6790690058","https://openalex.org/W6791705549","https://openalex.org/W6792695861","https://openalex.org/W6793632291","https://openalex.org/W6796494063","https://openalex.org/W6796580215","https://openalex.org/W6798837711","https://openalex.org/W6803028189","https://openalex.org/W6809995052"],"related_works":["https://openalex.org/W4315697128","https://openalex.org/W3102845713","https://openalex.org/W2971502891","https://openalex.org/W3205506801","https://openalex.org/W4388335561","https://openalex.org/W4280599700","https://openalex.org/W2970530566","https://openalex.org/W2798769807","https://openalex.org/W4304700937","https://openalex.org/W3036852732"],"abstract_inverted_index":{"In":[0],"this":[1,47],"paper,":[2],"we":[3,49,72,116,161,195],"present":[4],"a":[5,31,51,74,128,163,266,303],"new":[6,304],"approach":[7],"for":[8,41,131,175,258,309],"model":[9,310],"acceleration":[10,268],"by":[11,106,247],"exploiting":[12],"spatial":[13,166,177,300],"sparsity":[14],"in":[15,24,113],"visual":[16,218],"data.":[17],"We":[18,135],"observe":[19],"that":[20,118,298],"the":[21,34,65,80,86,102,110,119,198,207,221,232,245,251],"final":[22],"prediction":[23,76,154],"vision":[25,69,114,146,260],"Transformers":[26,147],"is":[27,39,91,104,255,313],"only":[28],"based":[29,63],"on":[30,46,64,212,272,284],"subset":[32],"of":[33,82,109,121,201,223,231,253],"most":[35],"informative":[36,186],"regions,":[37],"which":[38],"sufficient":[40],"accurate":[42],"image":[43],"recognition.":[44],"Based":[45],"observation,":[48],"propose":[50],"dynamic":[52,165,299],"token":[53,84],"sparsification":[54,167,171,301],"framework":[55,103,168],"to":[56,67,78,93,96,139,184,192],"prune":[57,97],"redundant":[58,98],"tokens":[59,99],"progressively":[60],"and":[61,123,144,172,188,216,243,275,291,305],"dynamically":[62],"input":[66,233],"accelerate":[68],"Transformers.":[70,261,277],"Specifically,":[71],"devise":[73],"lightweight":[75,181],"module":[77,90],"estimate":[79],"importance":[81],"each":[83],"given":[85],"current":[87],"features.":[88],"The":[89],"added":[92],"different":[94,176,217],"layers":[95],"hierarchically.":[100],"While":[101],"inspired":[105],"our":[107,137,224,235,279],"observation":[108],"sparse":[111],"attention":[112],"Transformers,":[115],"find":[117],"idea":[120],"adaptive":[122],"asymmetric":[124,173,264],"computation":[125,174],"can":[126,196,269],"be":[127,270],"general":[129],"solution":[130],"accelerating":[132],"various":[133,259],"architectures.":[134],"extend":[136],"method":[138,236,280],"hierarchical":[140,145],"models":[141],"including":[142,288],"CNNs":[143,274],"as":[148,150],"well":[149],"more":[151,285,306],"complex":[152,286],"dense":[153],"tasks.":[155],"To":[156],"handle":[157],"structured":[158],"feature":[159,202],"maps,":[160],"formulate":[162],"generic":[164],"with":[169],"progressive":[170],"locations.":[178],"By":[179,227,262],"applying":[180],"fast":[182],"paths":[183,191],"less":[185],"features":[187],"expressive":[189],"slow":[190],"important":[193],"locations,":[194],"maintain":[197],"complete":[199],"structure":[200],"maps":[203],"while":[204,250],"significantly":[205],"reducing":[206],"overall":[208],"computations.":[209],"Extensive":[210],"experiments":[211],"diverse":[213],"modern":[214,273],"architectures":[215],"tasks":[219,287],"demonstrate":[220,297],"effectiveness":[222],"proposed":[225],"framework.":[226],"hierarchically":[228],"pruning":[229],"66%":[230],"tokens,":[234],"greatly":[237],"reduces":[238],"31%":[239],"\u223c":[240],"35%":[241],"FLOPs":[242],"improves":[244],"throughput":[246],"over":[248],"40%":[249],"drop":[252],"accuracy":[254],"within":[256],"0.5%":[257],"introducing":[263],"computation,":[265],"similar":[267],"achieved":[271],"Swin":[276],"Moreover,":[278],"achieves":[281],"promising":[282],"results":[283,295],"semantic":[289],"segmentation":[290],"object":[292],"detection.":[293],"Our":[294],"clearly":[296],"offers":[302],"effective":[307],"dimension":[308],"acceleration.":[311],"Code":[312],"available":[314],"at":[315],"https://github.com/raoyongming/DynamicViT.":[316]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":26},{"year":2024,"cited_by_count":21},{"year":2023,"cited_by_count":6}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}