{"id":"https://openalex.org/W7165407286","doi":"https://doi.org/10.48550/arxiv.2606.19483","title":"LEAP: Layer-skipping Efficiency via Adaptive Progression for Vision Transformer Distillation","display_name":"LEAP: Layer-skipping Efficiency via Adaptive Progression for Vision Transformer Distillation","publication_year":2026,"publication_date":"2026-06-17","ids":{"openalex":"https://openalex.org/W7165407286","doi":"https://doi.org/10.48550/arxiv.2606.19483"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.19483","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.19483","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.19483","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138994703","display_name":"Jiaqi Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Jiaqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065422556","display_name":"Ashton Lee","orcid":"https://orcid.org/0000-0001-9471-5696"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Ashton","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135651933","display_name":"Anthony Wong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wong, Anthony","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139016208","display_name":"John Zou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zou, John","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138988396","display_name":"Sami BuGhanem","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"BuGhanem, Sami","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139024487","display_name":"Randall Balestriero","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Balestriero, Randall","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.8237000107765198,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.8237000107765198,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.049400001764297485,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.04039999842643738,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6067000031471252},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5357000231742859},{"id":"https://openalex.org/keywords/feature-learning","display_name":"Feature learning","score":0.42399999499320984},{"id":"https://openalex.org/keywords/curriculum","display_name":"Curriculum","score":0.41920000314712524},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.39640000462532043},{"id":"https://openalex.org/keywords/flops","display_name":"FLOPS","score":0.37229999899864197},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.36480000615119934},{"id":"https://openalex.org/keywords/sort","display_name":"sort","score":0.33820000290870667}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7088000178337097},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6067000031471252},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5627999901771545},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5357000231742859},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5192999839782715},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.42399999499320984},{"id":"https://openalex.org/C47177190","wikidata":"https://www.wikidata.org/wiki/Q207137","display_name":"Curriculum","level":2,"score":0.41920000314712524},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.39640000462532043},{"id":"https://openalex.org/C3826847","wikidata":"https://www.wikidata.org/wiki/Q188768","display_name":"FLOPS","level":2,"score":0.37229999899864197},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.36480000615119934},{"id":"https://openalex.org/C88548561","wikidata":"https://www.wikidata.org/wiki/Q347599","display_name":"sort","level":2,"score":0.33820000290870667},{"id":"https://openalex.org/C161301231","wikidata":"https://www.wikidata.org/wiki/Q3478658","display_name":"Knowledge representation and reasoning","level":2,"score":0.3278000056743622},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3084000051021576},{"id":"https://openalex.org/C138236772","wikidata":"https://www.wikidata.org/wiki/Q25098575","display_name":"Edge device","level":3,"score":0.2994999885559082},{"id":"https://openalex.org/C64876066","wikidata":"https://www.wikidata.org/wiki/Q5141226","display_name":"Cognitive neuroscience of visual object recognition","level":3,"score":0.29490000009536743},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.29319998621940613},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.28529998660087585},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.27950000762939453},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2768999934196472},{"id":"https://openalex.org/C5339829","wikidata":"https://www.wikidata.org/wiki/Q1425977","display_name":"Machine vision","level":2,"score":0.259799987077713},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.2574000060558319},{"id":"https://openalex.org/C162307627","wikidata":"https://www.wikidata.org/wiki/Q204833","display_name":"Enhanced Data Rates for GSM Evolution","level":2,"score":0.2540000081062317}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.19483","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.19483","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.19483","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.19483","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.726408064365387,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Vision":[0,5],"Foundation":[1],"Models":[2],"(VFMs)":[3],"with":[4,148],"Transformer":[6],"(ViT)":[7],"backbones,":[8],"such":[9],"as":[10,90],"DINOv2,":[11],"have":[12],"become":[13],"essential":[14],"for":[15,36,78,158,190],"downstream":[16],"tasks":[17],"like":[18],"object":[19],"recognition":[20],"and":[21,130,155,166,179],"semantic":[22],"segmentation.":[23],"The":[24],"immense":[25],"computational":[26],"requirements":[27],"of":[28,93,197],"backbones":[29],"often":[30,43],"necessitate":[31],"distillation":[32,41],"into":[33],"smaller":[34],"architectures":[35],"edge":[37],"deployment.":[38],"Feature-based":[39],"knowledge":[40,81],"(KD)":[42],"suffers":[44],"from":[45],"the":[46,49,85,101,136,159,164,171,194],"teacher-student":[47],"gap;":[48],"student":[50,102,127],"struggles":[51],"to":[52,59,103],"imitate":[53],"teacher's":[54,86],"complex":[55],"feature":[56,88],"map":[57],"due":[58],"its":[60],"limited":[61],"capacity.":[62],"To":[63],"mitigate":[64],"this":[65,116],"bottleneck,":[66],"we":[67],"propose":[68],"LEAP:":[69],"Layer-skipping":[70],"Efficiency":[71],"via":[72],"Adaptive":[73],"Progression,":[74],"a":[75,91,105,144],"training":[76,177,183],"curriculum":[77,99,172],"ViT":[79],"feature-based":[80],"distillation.":[82],"By":[83],"utilizing":[84],"intermediate":[87],"maps":[89],"sequence":[92],"progressively":[94],"more":[95],"difficult":[96],"targets,":[97],"our":[98,134],"allows":[100],"build":[104],"foundational":[106],"representation":[107],"before":[108],"tackling":[109],"higher-level":[110],"abstractions.":[111],"Our":[112],"results":[113],"demonstrate":[114],"that":[115],"paradigm":[117],"significantly":[118],"accelerates":[119],"convergence":[120],"through":[121],"adaptive":[122],"difficulty":[123],"selection":[124],"across":[125],"various":[126],"model":[128],"sizes":[129],"dataset":[131],"scales.":[132],"With":[133],"curriculum,":[135],"LEAP-distilled":[137],"ViT-S":[138],"achieves":[139,153],"90.1%":[140],"accuracy":[141],"on":[142,163,185],"ImageNet-100,":[143],"+12.24%":[145],"improvement":[146,157],"compared":[147],"baseline.":[149],"On":[150],"ImageNet-1K,":[151],"LEAP":[152],"+3.84%":[154],"+7.75%":[156],"instance":[160],"retrieval":[161],"task":[162],"Oxford":[165],"Paris":[167],"datasets,":[168],"respectively.":[169],"Furthermore,":[170],"enables":[173],"25.1%":[174],"savings":[175,181],"in":[176,182],"FLOPs":[178],"21%":[180],"time":[184],"ImageNet-100":[186],"by":[187],"implementing":[188],"early-stopping":[189],"teacher":[191],"inference":[192],"during":[193],"initial":[195],"stages":[196],"training.":[198],"Code":[199],"is":[200],"available":[201],"at":[202],"https://github.com/KevinZ0217/LEAP":[203]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-20T00:00:00"}