{"id":"https://openalex.org/W7129049366","doi":"https://doi.org/10.1111/coin.70174","title":"Optimizing Vision Transformers With Data\u2010Free Knowledge Transfer","display_name":"Optimizing Vision Transformers With Data\u2010Free Knowledge Transfer","publication_year":2026,"publication_date":"2026-02-01","ids":{"openalex":"https://openalex.org/W7129049366","doi":"https://doi.org/10.1111/coin.70174"},"language":"en","primary_location":{"id":"doi:10.1111/coin.70174","doi":"https://doi.org/10.1111/coin.70174","is_oa":false,"landing_page_url":"https://doi.org/10.1111/coin.70174","pdf_url":null,"source":{"id":"https://openalex.org/S56561474","display_name":"Computational Intelligence","issn_l":"0824-7935","issn":["0824-7935","1467-8640"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-c","cwts-core","dongbi-d","jufo-1","poland-100","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Intelligence","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2408.05952","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041640551","display_name":"Gousia Habib","orcid":"https://orcid.org/0000-0002-4773-3733","observed_orcids":["https://orcid.org/0000-0002-4773-3733"]},"institutions":[{"id":"https://openalex.org/I4210133311","display_name":"Shoolini University","ror":"https://ror.org/02xe2fg84","country_code":"IN","type":"education","lineage":["https://openalex.org/I4210133311"]},{"id":"https://openalex.org/I68891433","display_name":"Indian Institute of Technology Delhi","ror":"https://ror.org/049tgcd06","country_code":"IN","type":"education","lineage":["https://openalex.org/I68891433"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Gousia Habib","raw_affiliation_strings":["Bharti School of Telcom Technology and Management Indian Institute of Technology Delhi  New Delhi India","Yogananda School of AI, Computer and Data Science Shoolini University  Solan Himachal Pradesh India"],"raw_orcid":"https://orcid.org/0000-0002-4773-3733","affiliations":[{"raw_affiliation_string":"Bharti School of Telcom Technology and Management Indian Institute of Technology Delhi  New Delhi India","institution_ids":["https://openalex.org/I68891433"]},{"raw_affiliation_string":"Yogananda School of AI, Computer and Data Science Shoolini University  Solan Himachal Pradesh India","institution_ids":["https://openalex.org/I4210133311"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5161734627","display_name":"Damandeep Singh","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I68891433","display_name":"Indian Institute of Technology Delhi","ror":"https://ror.org/049tgcd06","country_code":"IN","type":"education","lineage":["https://openalex.org/I68891433"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Damandeep Singh","raw_affiliation_strings":["Bharti School of Telcom Technology and Management Indian Institute of Technology Delhi  New Delhi India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bharti School of Telcom Technology and Management Indian Institute of Technology Delhi  New Delhi India","institution_ids":["https://openalex.org/I68891433"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061041359","display_name":"Ishfaq Ahmad Malik","orcid":"https://orcid.org/0000-0002-9634-374X","observed_orcids":["https://orcid.org/0000-0002-9634-374X"]},"institutions":[{"id":"https://openalex.org/I4210133311","display_name":"Shoolini University","ror":"https://ror.org/02xe2fg84","country_code":"IN","type":"education","lineage":["https://openalex.org/I4210133311"]}],"countries":["IN"],"is_corresponding":true,"raw_author_name":"Ishfaq Ahmad Malik","raw_affiliation_strings":["Yogananda School of AI, Computer and Data Science Shoolini University  Solan Himachal Pradesh India"],"raw_orcid":"https://orcid.org/0000-0002-9634-374X","affiliations":[{"raw_affiliation_string":"Yogananda School of AI, Computer and Data Science Shoolini University  Solan Himachal Pradesh India","institution_ids":["https://openalex.org/I4210133311"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5061041359"],"corresponding_institution_ids":["https://openalex.org/I4210133311"],"apc_list":{"value":2860,"currency":"USD","value_usd":2860},"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":97},"biblio":{"volume":"42","issue":"1","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/vision-transformer","display_name":"vision transformer","score":0.9950000047683716},{"id":"https://openalex.org/keywords/data-free-knowledge-distillation","display_name":"data-free knowledge distillation","score":0.921999990940094},{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.878000020980835},{"id":"https://openalex.org/keywords/resource-constrained-devices","display_name":"resource-constrained devices","score":0.6660000085830688},{"id":"https://openalex.org/keywords/object-detection","display_name":"object detection","score":0.7860000133514404},{"id":"https://openalex.org/keywords/image-classification","display_name":"image classification","score":0.9150000214576721},{"id":"https://openalex.org/keywords/self-attention","display_name":"self-attention","score":0.8209999799728394},{"id":"https://openalex.org/keywords/edge-deployment","display_name":"edge deployment","score":0.43700000643730164}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8097000122070312},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6862000226974487},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5328999757766724},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.5112000107765198},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.4961000084877014},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.45840001106262207},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3765000104904175},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.375},{"id":"https://openalex.org/C64876066","wikidata":"https://www.wikidata.org/wiki/Q5141226","display_name":"Cognitive neuroscience of visual object recognition","level":3,"score":0.3709999918937683},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.349700003862381},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.3325999975204468},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.3253999948501587},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.3246000111103058},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3084999918937683},{"id":"https://openalex.org/C2776151529","wikidata":"https://www.wikidata.org/wiki/Q3045304","display_name":"Object detection","level":3,"score":0.2921000123023987},{"id":"https://openalex.org/C5339829","wikidata":"https://www.wikidata.org/wiki/Q1425977","display_name":"Machine vision","level":2,"score":0.2718000113964081},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.26010000705718994},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.2524000108242035}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1111/coin.70174","doi":"https://doi.org/10.1111/coin.70174","is_oa":false,"landing_page_url":"https://doi.org/10.1111/coin.70174","pdf_url":null,"source":{"id":"https://openalex.org/S56561474","display_name":"Computational Intelligence","issn_l":"0824-7935","issn":["0824-7935","1467-8640"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-c","cwts-core","dongbi-d","jufo-1","poland-100","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2408.05952","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.05952","pdf_url":"https://arxiv.org/pdf/2408.05952","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2408.05952","doi":"https://doi.org/10.48550/arxiv.2408.05952","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2408.05952","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2408.05952","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.05952","pdf_url":"https://arxiv.org/pdf/2408.05952","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.428600013256073,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7129049366.pdf","grobid_xml":"https://content.openalex.org/works/W7129049366.grobid-xml"},"referenced_works_count":36,"referenced_works":["https://openalex.org/W1821462560","https://openalex.org/W1861492603","https://openalex.org/W2782980316","https://openalex.org/W3030520226","https://openalex.org/W3034368386","https://openalex.org/W3034957837","https://openalex.org/W3046336220","https://openalex.org/W3094297360","https://openalex.org/W3094502228","https://openalex.org/W3116489684","https://openalex.org/W3119373805","https://openalex.org/W3132401450","https://openalex.org/W3187295906","https://openalex.org/W3191459272","https://openalex.org/W3204628622","https://openalex.org/W4206419470","https://openalex.org/W4352978394","https://openalex.org/W4386065380","https://openalex.org/W4386075852","https://openalex.org/W4388923662","https://openalex.org/W4390548081","https://openalex.org/W4390873603","https://openalex.org/W4392019445","https://openalex.org/W4392341757","https://openalex.org/W4392397478","https://openalex.org/W4392867127","https://openalex.org/W4393154018","https://openalex.org/W4400204161","https://openalex.org/W4402667890","https://openalex.org/W4402716099","https://openalex.org/W4402917080","https://openalex.org/W4403221502","https://openalex.org/W4404599563","https://openalex.org/W4404931559","https://openalex.org/W4415797115","https://openalex.org/W4415797161"],"related_works":["https://openalex.org/W4319452261","https://openalex.org/W4386555984","https://openalex.org/W4312671791","https://openalex.org/W4312677967","https://openalex.org/W4226484461","https://openalex.org/W7133185110","https://openalex.org/W4400366382","https://openalex.org/W4293029286","https://openalex.org/W4296499650","https://openalex.org/W4402916916"],"abstract_inverted_index":{"ABSTRACT":[0],"The":[1],"groundbreaking":[2],"performance":[3,99,107,147],"of":[4,17,43,120,141,171,245],"transformers":[5,44,56],"in":[6,45,60,159,219],"Natural":[7],"Language":[8],"Processing":[9],"(NLP)":[10],"tasks":[11,48,64],"has":[12,36],"led":[13],"to":[14,24,39,49,66,70,101,115,145,149,184,201,205,221,238],"their":[15,67,105,123],"replacement":[16],"traditional":[18,102],"Convolutional":[19],"Neural":[20],"Networks":[21],"(CNNs),":[22],"owing":[23],"the":[25,31,41,75,167,175,216,243],"efficiency":[26],"and":[27,92,174],"accuracy":[28],"achieved":[29],"through":[30],"self\u2010attention":[32,76],"mechanism.":[33,77],"This":[34],"success":[35],"inspired":[37],"researchers":[38],"explore":[40],"use":[42],"computer":[46,62],"vision":[47,63],"attain":[50],"enhanced":[51],"long\u2010term":[52],"semantic":[53,90],"awareness.":[54],"Vision":[55],"(ViTs)":[57],"have":[58],"excelled":[59],"various":[61],"due":[65,114],"superior":[68],"ability":[69],"capture":[71],"long\u2010distance":[72],"dependencies":[73],"using":[74,193],"Contemporary":[78],"ViTs":[79,136,161,246],"like":[80,130],"Data":[81],"Efficient":[82],"Transformers":[83],"(DeiT)":[84],"can":[85],"effectively":[86],"learn":[87],"both":[88,240],"global":[89],"information":[91,95],"local":[93],"texture":[94],"from":[96],"images,":[97],"achieving":[98],"comparable":[100,148],"CNNs.":[103],"However,":[104],"impressive":[106],"comes":[108],"with":[109,127],"a":[110,116,138,182],"high":[111,168],"computational":[112,169],"cost":[113],"very":[117],"large":[118,139,172,190],"number":[119],"parameters,":[121],"hindering":[122],"deployment":[124,244],"on":[125,162,212,225,247],"devices":[126],"limited":[128],"resources":[129],"smartphones,":[131],"cameras,":[132],"drones,":[133],"etc.":[134],"Additionally,":[135,208],"require":[137],"amount":[140],"data":[142,206],"for":[143,177],"training":[144,179],"achieve":[146],"benchmark":[150],"CNN":[151],"models.":[152],"Therefore,":[153],"we":[154,187,209,228],"identified":[155],"two":[156],"key":[157],"challenges":[158],"deploying":[160],"smaller":[163],"form":[164],"factor":[165],"devices:":[166],"requirements":[170],"models":[173,192],"need":[176],"extensive":[178],"data.":[180],"As":[181],"solution":[183],"these":[185],"challenges,":[186],"propose":[188],"compressing":[189],"ViT":[191],"Knowledge":[194],"Distillation":[195],"(KD),":[196],"which":[197],"is":[198,234],"implemented":[199],"data\u2010free":[200,231],"circumvent":[202],"limitations":[203],"related":[204],"availability.":[207],"conducted":[210],"experiments":[211],"object":[213],"detection":[214],"within":[215],"same":[217],"environment":[218],"addition":[220],"classification":[222],"tasks.":[223],"Based":[224],"our":[226],"analysis,":[227],"found":[229],"that":[230],"knowledge":[232],"distillation":[233],"an":[235],"effective":[236],"method":[237],"overcome":[239],"issues,":[241],"enabling":[242],"less":[248],"resource\u2010constrained":[249],"devices.":[250]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-13T13:42:25.865203","created_date":"2026-02-17T00:00:00"}