{"id":"https://openalex.org/W4402915866","doi":"https://doi.org/10.1109/icip51287.2024.10647295","title":"Vito: Vision Transformer Optimization Via Knowledge Distillation On Decoders","display_name":"Vito: Vision Transformer Optimization Via Knowledge Distillation On Decoders","publication_year":2024,"publication_date":"2024-09-27","ids":{"openalex":"https://openalex.org/W4402915866","doi":"https://doi.org/10.1109/icip51287.2024.10647295"},"language":"en","primary_location":{"id":"doi:10.1109/icip51287.2024.10647295","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/icip51287.2024.10647295","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Image Processing (ICIP)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083136342","display_name":"Giovanni Bellitto","orcid":"https://orcid.org/0000-0002-1333-8348"},"institutions":[{"id":"https://openalex.org/I39063666","display_name":"University of Catania","ror":"https://ror.org/03a64bh57","country_code":"IT","type":"education","lineage":["https://openalex.org/I39063666"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Giovanni Bellitto","raw_affiliation_strings":["University of Catania,PeRCeiVe Lab,Italy"],"affiliations":[{"raw_affiliation_string":"University of Catania,PeRCeiVe Lab,Italy","institution_ids":["https://openalex.org/I39063666"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004240376","display_name":"Renato Sortino","orcid":"https://orcid.org/0000-0002-3906-797X"},"institutions":[{"id":"https://openalex.org/I39063666","display_name":"University of Catania","ror":"https://ror.org/03a64bh57","country_code":"IT","type":"education","lineage":["https://openalex.org/I39063666"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Renato Sortino","raw_affiliation_strings":["University of Catania,PeRCeiVe Lab,Italy"],"affiliations":[{"raw_affiliation_string":"University of Catania,PeRCeiVe Lab,Italy","institution_ids":["https://openalex.org/I39063666"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018526690","display_name":"Paolo Spadaro","orcid":null},"institutions":[{"id":"https://openalex.org/I39063666","display_name":"University of Catania","ror":"https://ror.org/03a64bh57","country_code":"IT","type":"education","lineage":["https://openalex.org/I39063666"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Paolo Spadaro","raw_affiliation_strings":["University of Catania,PeRCeiVe Lab,Italy"],"affiliations":[{"raw_affiliation_string":"University of Catania,PeRCeiVe Lab,Italy","institution_ids":["https://openalex.org/I39063666"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031450889","display_name":"Simone Palazzo","orcid":"https://orcid.org/0000-0002-2441-0982"},"institutions":[{"id":"https://openalex.org/I39063666","display_name":"University of Catania","ror":"https://ror.org/03a64bh57","country_code":"IT","type":"education","lineage":["https://openalex.org/I39063666"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Simone Palazzo","raw_affiliation_strings":["University of Catania,PeRCeiVe Lab,Italy"],"affiliations":[{"raw_affiliation_string":"University of Catania,PeRCeiVe Lab,Italy","institution_ids":["https://openalex.org/I39063666"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033378482","display_name":"Federica Proietto Salanitri","orcid":"https://orcid.org/0000-0002-6122-4249"},"institutions":[{"id":"https://openalex.org/I39063666","display_name":"University of Catania","ror":"https://ror.org/03a64bh57","country_code":"IT","type":"education","lineage":["https://openalex.org/I39063666"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Federica Proietto Salanitri","raw_affiliation_strings":["University of Catania,PeRCeiVe Lab,Italy"],"affiliations":[{"raw_affiliation_string":"University of Catania,PeRCeiVe Lab,Italy","institution_ids":["https://openalex.org/I39063666"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046146028","display_name":"Giuseppe Fiameni","orcid":"https://orcid.org/0000-0001-8687-6609"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Giuseppe Fiameni","raw_affiliation_strings":["NVIDIA"],"affiliations":[{"raw_affiliation_string":"NVIDIA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002625178","display_name":"Efstratios Gavves","orcid":"https://orcid.org/0000-0001-8947-1332"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Efstratios Gavves","raw_affiliation_strings":["University of Amsterdam,Informatics Institute,The Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam,Informatics Institute,The Netherlands","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075815307","display_name":"Concetto Spampinato","orcid":"https://orcid.org/0000-0001-6653-2577"},"institutions":[{"id":"https://openalex.org/I39063666","display_name":"University of Catania","ror":"https://ror.org/03a64bh57","country_code":"IT","type":"education","lineage":["https://openalex.org/I39063666"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Concetto Spampinato","raw_affiliation_strings":["University of Catania,PeRCeiVe Lab,Italy"],"affiliations":[{"raw_affiliation_string":"University of Catania,PeRCeiVe Lab,Italy","institution_ids":["https://openalex.org/I39063666"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5083136342"],"corresponding_institution_ids":["https://openalex.org/I39063666"],"apc_list":null,"apc_paid":null,"fwci":0.3383,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.64112953,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"493","last_page":"499"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13114","display_name":"Image Processing Techniques and Applications","score":0.9438999891281128,"subfield":{"id":"https://openalex.org/subfields/2214","display_name":"Media Technology"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13114","display_name":"Image Processing Techniques and Applications","score":0.9438999891281128,"subfield":{"id":"https://openalex.org/subfields/2214","display_name":"Media Technology"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10299","display_name":"Photonic and Optical Devices","score":0.9106000065803528,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.706293523311615},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.6603561639785767},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6105765700340271},{"id":"https://openalex.org/keywords/engineering-drawing","display_name":"Engineering drawing","score":0.3359622359275818},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3302372097969055},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.22894799709320068},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.22069326043128967},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.15985551476478577},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.10623541474342346},{"id":"https://openalex.org/keywords/chromatography","display_name":"Chromatography","score":0.07664221525192261}],"concepts":[{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.706293523311615},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.6603561639785767},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6105765700340271},{"id":"https://openalex.org/C199639397","wikidata":"https://www.wikidata.org/wiki/Q1788588","display_name":"Engineering drawing","level":1,"score":0.3359622359275818},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3302372097969055},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.22894799709320068},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.22069326043128967},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.15985551476478577},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.10623541474342346},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.07664221525192261}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icip51287.2024.10647295","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/icip51287.2024.10647295","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Image Processing (ICIP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":40,"referenced_works":["https://openalex.org/W1690739335","https://openalex.org/W2117539524","https://openalex.org/W2407386500","https://openalex.org/W2950784811","https://openalex.org/W2964111476","https://openalex.org/W2978017171","https://openalex.org/W3001279689","https://openalex.org/W3034429256","https://openalex.org/W3096609285","https://openalex.org/W3105966348","https://openalex.org/W3146365155","https://openalex.org/W3159481202","https://openalex.org/W3164024107","https://openalex.org/W3169575272","https://openalex.org/W3187295906","https://openalex.org/W3217804443","https://openalex.org/W4214893857","https://openalex.org/W4226426325","https://openalex.org/W4287101401","https://openalex.org/W4287773438","https://openalex.org/W4289600407","https://openalex.org/W4312671791","https://openalex.org/W4312957757","https://openalex.org/W4321151009","https://openalex.org/W4385245566","https://openalex.org/W6637551013","https://openalex.org/W6739901393","https://openalex.org/W6768086466","https://openalex.org/W6768851824","https://openalex.org/W6772383348","https://openalex.org/W6778751070","https://openalex.org/W6779879114","https://openalex.org/W6784333009","https://openalex.org/W6788135285","https://openalex.org/W6790978476","https://openalex.org/W6795475546","https://openalex.org/W6811002592","https://openalex.org/W6840352845","https://openalex.org/W6841524142","https://openalex.org/W6842607031"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"In":[0],"this":[1,105],"paper,":[2],"we":[3],"propose":[4],"ViTO,":[5],"a":[6,15,19,40,71],"novel":[7],"knowledge":[8,141],"distillation":[9,142,165],"strategy":[10,106],"that":[11,22,44,57],"aims":[12,58],"to":[13,59,83,108,112,122],"convert":[14],"CNN":[16],"model":[17,48,56,75,150],"into":[18],"transformer-based":[20,149],"counterpart":[21],"incorporates":[23],"the":[24,61,73,77,84,89,99,110,160],"advantages":[25],"of":[26,104,162],"transformers":[27],"while":[28],"retaining":[29],"or":[30],"improving":[31],"its":[32],"inductive":[33],"bias.":[34],"Our":[35],"approach":[36],"is":[37,107],"based":[38],"on":[39,94,132,144],"two-level":[41],"transformer":[42,111],"architecture":[43],"includes":[45],"an":[46,54,68],"inner":[47],"for":[49],"learning":[50],"visual":[51,86],"representations":[52],"and":[53,115,126],"outer":[55,74],"match":[60],"teacher\u2019s":[62],"predictions":[63,90],"through":[64],"autoregression.":[65],"Specifically,":[66],"given":[67],"image":[69,78],"in":[70,81],"batch,":[72],"classifies":[76],"by":[79],"using,":[80],"addition":[82],"image\u2019s":[85],"properties,":[87],"also":[88],"it":[91],"has":[92],"made":[93],"images":[95,121],"previously":[96],"seen":[97],"within":[98],"same":[100],"batch.":[101],"The":[102],"effect":[103],"allow":[109],"estimate":[113],"self-":[114],"cross-attention":[116],"across":[117],"all":[118],"input":[119],"batch":[120],"learn":[123],"autoregressively":[124],"intra-class":[125],"inter-class":[127],"correlations.We":[128],"experimentally":[129],"validate":[130],"ViTO":[131],"several":[133],"standard":[134,156],"benchmarks":[135],"obtaining":[136],"better":[137,152],"performance":[138],"than":[139,155],"existing":[140],"strategies":[143],"transformers.":[145],"Furthermore,":[146],"our":[147,163],"distilled":[148],"shows":[151],"robustness":[153],"properties":[154],"vision":[157],"transformers,":[158],"demonstrating":[159],"effectiveness":[161],"proposed":[164],"strategy.":[166]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2025-12-21T01:58:51.020947","created_date":"2025-10-10T00:00:00"}