{"id":"https://openalex.org/W4224917841","doi":"https://doi.org/10.1109/icassp43922.2022.9747484","title":"Attention Probe: Vision Transformer Distillation in the Wild","display_name":"Attention Probe: Vision Transformer Distillation in the Wild","publication_year":2022,"publication_date":"2022-04-27","ids":{"openalex":"https://openalex.org/W4224917841","doi":"https://doi.org/10.1109/icassp43922.2022.9747484"},"language":"en","primary_location":{"id":"doi:10.1109/icassp43922.2022.9747484","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp43922.2022.9747484","pdf_url":null,"source":{"id":"https://openalex.org/S4363607702","display_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5108891681","display_name":"Jiahao Wang","orcid":"https://orcid.org/0009-0000-5496-9720"},"institutions":[{"id":"https://openalex.org/I4210114105","display_name":"Tsinghua\u2013Berkeley Shenzhen Institute","ror":"https://ror.org/02hhwwz98","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210114105","https://openalex.org/I95457486","https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Jiahao Wang","raw_affiliation_strings":["Tsinghua Shenzhen International Graduate School,Shenzhen,China","Tsinghua Shenzhen International Graduate School, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School,Shenzhen,China","institution_ids":["https://openalex.org/I4210114105"]},{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School, Shenzhen, China","institution_ids":["https://openalex.org/I4210114105"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013983776","display_name":"Mingdeng Cao","orcid":"https://orcid.org/0000-0002-6577-4715"},"institutions":[{"id":"https://openalex.org/I4210114105","display_name":"Tsinghua\u2013Berkeley Shenzhen Institute","ror":"https://ror.org/02hhwwz98","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210114105","https://openalex.org/I95457486","https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingdeng Cao","raw_affiliation_strings":["Tsinghua Shenzhen International Graduate School,Shenzhen,China","Tsinghua Shenzhen International Graduate School, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School,Shenzhen,China","institution_ids":["https://openalex.org/I4210114105"]},{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School, Shenzhen, China","institution_ids":["https://openalex.org/I4210114105"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086329538","display_name":"Shuwei Shi","orcid":null},"institutions":[{"id":"https://openalex.org/I4210114105","display_name":"Tsinghua\u2013Berkeley Shenzhen Institute","ror":"https://ror.org/02hhwwz98","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210114105","https://openalex.org/I95457486","https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuwei Shi","raw_affiliation_strings":["Tsinghua Shenzhen International Graduate School,Shenzhen,China","Tsinghua Shenzhen International Graduate School, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School,Shenzhen,China","institution_ids":["https://openalex.org/I4210114105"]},{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School, Shenzhen, China","institution_ids":["https://openalex.org/I4210114105"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068027800","display_name":"Baoyuan Wu","orcid":"https://orcid.org/0000-0003-2183-5990"},"institutions":[{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Baoyuan Wu","raw_affiliation_strings":["The Chinese University of Hong Kong,Shenzhen,China","The Chinese University of Hong Kong, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong,Shenzhen,China","institution_ids":["https://openalex.org/I4210116924"]},{"raw_affiliation_string":"The Chinese University of Hong Kong, Shenzhen, China","institution_ids":["https://openalex.org/I4210116924"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5020953714","display_name":"Yujiu Yang","orcid":"https://orcid.org/0000-0002-6427-1024"},"institutions":[{"id":"https://openalex.org/I4210114105","display_name":"Tsinghua\u2013Berkeley Shenzhen Institute","ror":"https://ror.org/02hhwwz98","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210114105","https://openalex.org/I95457486","https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yujiu Yang","raw_affiliation_strings":["Tsinghua Shenzhen International Graduate School,Shenzhen,China","Tsinghua Shenzhen International Graduate School, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School,Shenzhen,China","institution_ids":["https://openalex.org/I4210114105"]},{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School, Shenzhen, China","institution_ids":["https://openalex.org/I4210114105"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5108891681"],"corresponding_institution_ids":["https://openalex.org/I4210114105"],"apc_list":null,"apc_paid":null,"fwci":0.4797,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.72294311,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"2220","last_page":"2224"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11992","display_name":"CCD and CMOS Imaging Sensors","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8259507417678833},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.8204082250595093},{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.6410941481590271},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.5565290451049805},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5478675961494446},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.4911594092845917},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4780198335647583},{"id":"https://openalex.org/keywords/mobile-device","display_name":"Mobile device","score":0.46657097339630127},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4182053804397583},{"id":"https://openalex.org/keywords/data-modeling","display_name":"Data modeling","score":0.411212682723999},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.3538729250431061},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.34660178422927856},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.323071151971817},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.160739928483963},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.15926480293273926}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8259507417678833},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.8204082250595093},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.6410941481590271},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5565290451049805},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5478675961494446},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.4911594092845917},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4780198335647583},{"id":"https://openalex.org/C186967261","wikidata":"https://www.wikidata.org/wiki/Q5082128","display_name":"Mobile device","level":2,"score":0.46657097339630127},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4182053804397583},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.411212682723999},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.3538729250431061},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.34660178422927856},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.323071151971817},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.160739928483963},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.15926480293273926},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp43922.2022.9747484","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp43922.2022.9747484","pdf_url":null,"source":{"id":"https://openalex.org/S4363607702","display_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":36,"referenced_works":["https://openalex.org/W1821462560","https://openalex.org/W2117539524","https://openalex.org/W2119144962","https://openalex.org/W2155803963","https://openalex.org/W2554242204","https://openalex.org/W2734358244","https://openalex.org/W2766966408","https://openalex.org/W2908510526","https://openalex.org/W2962965870","https://openalex.org/W2963122961","https://openalex.org/W2964217848","https://openalex.org/W2982802130","https://openalex.org/W2997563695","https://openalex.org/W3034957837","https://openalex.org/W3094502228","https://openalex.org/W3096609285","https://openalex.org/W3167272120","https://openalex.org/W3170874841","https://openalex.org/W3171125843","https://openalex.org/W3176592254","https://openalex.org/W3207918547","https://openalex.org/W3212214438","https://openalex.org/W4312812783","https://openalex.org/W4385245566","https://openalex.org/W6638523607","https://openalex.org/W6677580257","https://openalex.org/W6726275242","https://openalex.org/W6729623471","https://openalex.org/W6739901393","https://openalex.org/W6745689806","https://openalex.org/W6757817989","https://openalex.org/W6778485988","https://openalex.org/W6784333009","https://openalex.org/W6788135285","https://openalex.org/W6796298902","https://openalex.org/W6804296794"],"related_works":["https://openalex.org/W3034529322","https://openalex.org/W2113597336","https://openalex.org/W2115913271","https://openalex.org/W2155505549","https://openalex.org/W2357479218","https://openalex.org/W1819546284","https://openalex.org/W2043253324","https://openalex.org/W3026162553","https://openalex.org/W2018648706","https://openalex.org/W4389518428"],"abstract_inverted_index":{"Vision":[0],"transformers":[1],"(ViTs)":[2],"require":[3],"intensive":[4],"computational":[5],"resources":[6],"to":[7,24,38,72,106,150],"achieve":[8,193],"high":[9],"performance,":[10],"which":[11,32,59],"usually":[12],"makes":[13],"them":[14,26],"not":[15,35],"suitable":[16,98],"for":[17,64,99],"mobile":[18],"devices.":[19],"A":[20],"feasible":[21],"strategy":[22],"is":[23,55,87,97,206],"compress":[25,108],"using":[27,110],"the":[28,48,53,73,94,111,115,132,140,158,162,168,172,184,189,197,201],"original":[29,202],"training":[30,203],"data,":[31,142],"may":[33],"be":[34],"accessible":[36],"due":[37,71],"privacy":[39],"limitations":[40],"or":[41],"transmission":[42],"restrictions.":[43],"In":[44,101],"this":[45,102],"case,":[46],"utilizing":[47],"massive":[49],"unlabeled":[50,112],"data":[51,113,130],"in":[52,76,114,127],"wild":[54],"an":[56,89,124],"alternative":[57],"paradigm,":[58],"has":[60],"been":[61],"proved":[62],"effective":[63,125],"compressing":[65],"convolutional":[66],"neural":[67],"networks":[68],"(CNNs).":[69],"However,":[70],"significant":[74],"differences":[75],"model":[77],"structure":[78],"and":[79,84,164,171],"computation":[80],"mechanism":[81],"between":[82,167],"CNNs":[83],"ViTs,":[85],"it":[86],"still":[88],"open":[90],"issue":[91],"that":[92,183,199],"whether":[93],"similar":[95],"paradigm":[96],"ViTs.":[100],"work,":[103],"we":[104,122,143],"propose":[105],"effectively":[107],"ViTs":[109],"wild,":[116,133],"consisting":[117],"of":[118],"two":[119],"stages.":[120],"First,":[121],"design":[123],"tool":[126],"selecting":[128],"valuable":[129],"from":[131],"dubbed":[134],"Attention":[135],"Probe.":[136],"Second,":[137],"based":[138],"on":[139,160,179],"selected":[141],"develop":[144],"a":[145,152],"probe":[146],"knowledge":[147],"distillation":[148],"algorithm":[149],"train":[151],"lightweight":[153,173],"student":[154,174,185],"transformer,":[155],"through":[156],"maximizing":[157],"similarities":[159],"both":[161],"outputs":[163],"intermediate":[165],"features,":[166],"heavy":[169],"teacher":[170],"models.":[175],"Extensive":[176],"experimental":[177],"results":[178],"several":[180],"benchmarks":[181],"demonstrate":[182],"transformer":[186],"obtained":[187],"by":[188],"proposed":[190],"method":[191],"can":[192],"comparable":[194],"performance":[195],"with":[196],"baseline":[198],"requires":[200],"data.":[204],"Code":[205],"available":[207],"at:":[208],"https://github.com/IIGROUP/AttentionProbe.":[209]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}