{"id":"https://openalex.org/W4392909676","doi":"https://doi.org/10.1109/icassp48485.2024.10447853","title":"Distill Vision Transformers to CNNs via Teacher Collaboration","display_name":"Distill Vision Transformers to CNNs via Teacher Collaboration","publication_year":2024,"publication_date":"2024-03-18","ids":{"openalex":"https://openalex.org/W4392909676","doi":"https://doi.org/10.1109/icassp48485.2024.10447853"},"language":"en","primary_location":{"id":"doi:10.1109/icassp48485.2024.10447853","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10447853","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111339691","display_name":"Sunqi Lin","orcid":null},"institutions":[{"id":"https://openalex.org/I109935558","display_name":"Ningbo University","ror":"https://ror.org/03et85d35","country_code":"CN","type":"education","lineage":["https://openalex.org/I109935558"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Sunqi Lin","raw_affiliation_strings":["Ningbo University,Faculty of Electrical Engineering and Computer Science,China","Faculty of Electrical Engineering and Computer Science, Ningbo University, China"],"affiliations":[{"raw_affiliation_string":"Ningbo University,Faculty of Electrical Engineering and Computer Science,China","institution_ids":["https://openalex.org/I109935558"]},{"raw_affiliation_string":"Faculty of Electrical Engineering and Computer Science, Ningbo University, China","institution_ids":["https://openalex.org/I109935558"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100769643","display_name":"Chong Wang","orcid":"https://orcid.org/0000-0001-6016-6545"},"institutions":[{"id":"https://openalex.org/I109935558","display_name":"Ningbo University","ror":"https://ror.org/03et85d35","country_code":"CN","type":"education","lineage":["https://openalex.org/I109935558"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chong Wang","raw_affiliation_strings":["Ningbo University,Faculty of Electrical Engineering and Computer Science,China","Faculty of Electrical Engineering and Computer Science, Ningbo University, China","Zhejiang Engineering Research Center of Advcanced Mass Spectrometry and Clinical Application, China"],"affiliations":[{"raw_affiliation_string":"Ningbo University,Faculty of Electrical Engineering and Computer Science,China","institution_ids":["https://openalex.org/I109935558"]},{"raw_affiliation_string":"Faculty of Electrical Engineering and Computer Science, Ningbo University, China","institution_ids":["https://openalex.org/I109935558"]},{"raw_affiliation_string":"Zhejiang Engineering Research Center of Advcanced Mass Spectrometry and Clinical Application, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101441142","display_name":"Yujie Zheng","orcid":"https://orcid.org/0000-0001-5863-6843"},"institutions":[{"id":"https://openalex.org/I109935558","display_name":"Ningbo University","ror":"https://ror.org/03et85d35","country_code":"CN","type":"education","lineage":["https://openalex.org/I109935558"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yujie Zheng","raw_affiliation_strings":["Ningbo University,Faculty of Electrical Engineering and Computer Science,China","Faculty of Electrical Engineering and Computer Science, Ningbo University, China"],"affiliations":[{"raw_affiliation_string":"Ningbo University,Faculty of Electrical Engineering and Computer Science,China","institution_ids":["https://openalex.org/I109935558"]},{"raw_affiliation_string":"Faculty of Electrical Engineering and Computer Science, Ningbo University, China","institution_ids":["https://openalex.org/I109935558"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046569727","display_name":"Chenchen Tao","orcid":"https://orcid.org/0000-0003-2659-5359"},"institutions":[{"id":"https://openalex.org/I109935558","display_name":"Ningbo University","ror":"https://ror.org/03et85d35","country_code":"CN","type":"education","lineage":["https://openalex.org/I109935558"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chenchen Tao","raw_affiliation_strings":["Ningbo University,Faculty of Electrical Engineering and Computer Science,China","Faculty of Electrical Engineering and Computer Science, Ningbo University, China"],"affiliations":[{"raw_affiliation_string":"Ningbo University,Faculty of Electrical Engineering and Computer Science,China","institution_ids":["https://openalex.org/I109935558"]},{"raw_affiliation_string":"Faculty of Electrical Engineering and Computer Science, Ningbo University, China","institution_ids":["https://openalex.org/I109935558"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062686002","display_name":"Xinmiao Dai","orcid":"https://orcid.org/0009-0009-4559-7641"},"institutions":[{"id":"https://openalex.org/I109935558","display_name":"Ningbo University","ror":"https://ror.org/03et85d35","country_code":"CN","type":"education","lineage":["https://openalex.org/I109935558"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xinmiao Dai","raw_affiliation_strings":["Ningbo University,Faculty of Electrical Engineering and Computer Science,China","Faculty of Electrical Engineering and Computer Science, Ningbo University, China"],"affiliations":[{"raw_affiliation_string":"Ningbo University,Faculty of Electrical Engineering and Computer Science,China","institution_ids":["https://openalex.org/I109935558"]},{"raw_affiliation_string":"Faculty of Electrical Engineering and Computer Science, Ningbo University, China","institution_ids":["https://openalex.org/I109935558"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5030616168","display_name":"Yuqi Li","orcid":"https://orcid.org/0000-0003-0228-242X"},"institutions":[{"id":"https://openalex.org/I109935558","display_name":"Ningbo University","ror":"https://ror.org/03et85d35","country_code":"CN","type":"education","lineage":["https://openalex.org/I109935558"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuqi Li","raw_affiliation_strings":["Ningbo University,Faculty of Electrical Engineering and Computer Science,China","Faculty of Electrical Engineering and Computer Science, Ningbo University, China","Zhejiang Engineering Research Center of Advcanced Mass Spectrometry and Clinical Application, China"],"affiliations":[{"raw_affiliation_string":"Ningbo University,Faculty of Electrical Engineering and Computer Science,China","institution_ids":["https://openalex.org/I109935558"]},{"raw_affiliation_string":"Faculty of Electrical Engineering and Computer Science, Ningbo University, China","institution_ids":["https://openalex.org/I109935558"]},{"raw_affiliation_string":"Zhejiang Engineering Research Center of Advcanced Mass Spectrometry and Clinical Application, China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5111339691"],"corresponding_institution_ids":["https://openalex.org/I109935558"],"apc_list":null,"apc_paid":null,"fwci":0.7661,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.69815328,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"5925","last_page":"5929"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11992","display_name":"CCD and CMOS Imaging Sensors","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12389","display_name":"Infrared Target Detection Methodologies","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/2202","display_name":"Aerospace Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7696815729141235},{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.6799283623695374},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6654406785964966},{"id":"https://openalex.org/keywords/architecture","display_name":"Architecture","score":0.5904983282089233},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5686284303665161},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5187180638313293},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.42483580112457275},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.4197715222835541},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.4105921983718872},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3488870859146118},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.11604833602905273}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7696815729141235},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.6799283623695374},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6654406785964966},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.5904983282089233},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5686284303665161},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5187180638313293},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.42483580112457275},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4197715222835541},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.4105921983718872},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3488870859146118},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.11604833602905273},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp48485.2024.10447853","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10447853","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2087396116","display_name":null,"funder_award_id":"China","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3317480652","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4020255992","display_name":null,"funder_award_id":"Project","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4042026917","display_name":null,"funder_award_id":"No. 2022J114","funder_id":"https://openalex.org/F4320332587","funder_display_name":"Natural Science Foundation of Ningbo"},{"id":"https://openalex.org/G5994120800","display_name":null,"funder_award_id":"Natural","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320332587","display_name":"Natural Science Foundation of Ningbo","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W1690739335","https://openalex.org/W1821462560","https://openalex.org/W2300242332","https://openalex.org/W2605135468","https://openalex.org/W2887783173","https://openalex.org/W2963140444","https://openalex.org/W2982242214","https://openalex.org/W3094502228","https://openalex.org/W3096609285","https://openalex.org/W3160094467","https://openalex.org/W4224918845","https://openalex.org/W4226426325","https://openalex.org/W4281489207","https://openalex.org/W4305000041","https://openalex.org/W4372341472","https://openalex.org/W4379087175","https://openalex.org/W4382458595","https://openalex.org/W4390874575","https://openalex.org/W6637551013","https://openalex.org/W6638523607","https://openalex.org/W6769906912","https://openalex.org/W6784333009","https://openalex.org/W6788135285","https://openalex.org/W6846271418","https://openalex.org/W6853066206"],"related_works":["https://openalex.org/W4293226380","https://openalex.org/W4321487865","https://openalex.org/W4313906399","https://openalex.org/W3026162553","https://openalex.org/W4391266461","https://openalex.org/W2590798552","https://openalex.org/W2811106690","https://openalex.org/W4239306820","https://openalex.org/W2947043951","https://openalex.org/W4312417841"],"abstract_inverted_index":{"The":[0,107,120],"vision":[1],"transformer":[2],"(ViT)":[3],"has":[4],"recently":[5],"emerged":[6],"as":[7],"a":[8,68],"leading":[9],"approach":[10],"in":[11,47,61],"various":[12],"domains,":[13],"outperforming":[14],"other":[15],"methods.":[16],"Therefore,":[17],"it":[18],"is":[19,78,122],"logical":[20],"to":[21,32,43,80,96],"explore":[22],"the":[23,27,82,99,111],"possibility":[24],"of":[25],"transferring":[26],"superior":[28],"knowledge":[29,55,71],"from":[30],"ViT":[31,90],"more":[33],"compact":[34],"and":[35,49,91,104],"cost-effective":[36],"convolutional":[37],"neural":[38],"networks":[39],"(CNNs).":[40],"However,":[41],"due":[42],"substantial":[44],"architectural":[45],"disparities":[46],"representation":[48],"logits":[50],"between":[51],"these":[52],"models,":[53],"conventional":[54,115],"distillation":[56,72],"methods":[57,116],"have":[58],"proven":[59],"ineffective":[60],"this":[62,66],"context.":[63],"To":[64],"address":[65],"issue,":[67],"novel":[69],"cross-architecture":[70],"scheme":[73,113],"based":[74],"on":[75,117],"teacher":[76],"collaboration":[77],"proposed":[79,112],"alleviate":[81],"architecture":[83],"gap.":[84],"Two":[85],"different":[86],"teachers,":[87],"i.e.":[88],"one":[89,92],"CNN,":[93],"are":[94],"utilized":[95],"simultaneously":[97],"distill":[98],"student":[100],"by":[101],"feature":[102],"reaggregation":[103],"logit":[105],"correction.":[106],"experiments":[108],"show":[109],"that":[110],"outperforms":[114],"CIFAR-100":[118],"dataset.":[119],"code":[121],"available":[123],"at":[124],"https://github.com/SunkiLin/RCD.":[125]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2026-03-18T14:38:29.013473","created_date":"2025-10-10T00:00:00"}