{"id":"https://openalex.org/W7165867386","doi":"https://doi.org/10.48550/arxiv.2606.25488","title":"Distill on a Diet: Efficient Knowledge Distillation via Learnable Data Pruning","display_name":"Distill on a Diet: Efficient Knowledge Distillation via Learnable Data Pruning","publication_year":2026,"publication_date":"2026-06-24","ids":{"openalex":"https://openalex.org/W7165867386","doi":"https://doi.org/10.48550/arxiv.2606.25488"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.25488","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.25488","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.25488","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139342336","display_name":"Yifan Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Yifan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139375551","display_name":"Yiqi Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yiqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104135650","display_name":"Xichen Ye","orcid":"https://orcid.org/0009-0002-4041-2774"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ye, Xichen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139320539","display_name":"Wenjing Yan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yan, Wenjing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139316319","display_name":"Xiaoqiang Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Xiaoqiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139341102","display_name":"Cheng Jin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, Cheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139325183","display_name":"Xiangyu Yue","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yue, Xiangyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139361545","display_name":"Weizhong Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Weizhong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.5483999848365784,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.5483999848365784,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.1339000016450882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.11819999665021896,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.7473999857902527},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.5874000191688538},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.5455999970436096},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.49880000948905945},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.45190000534057617},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.43140000104904175},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4246000051498413}],"concepts":[{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.7473999857902527},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7301999926567078},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.5874000191688538},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.580299973487854},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.5455999970436096},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5149000287055969},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.49880000948905945},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.45190000534057617},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.43140000104904175},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4246000051498413},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.39149999618530273},{"id":"https://openalex.org/C165464430","wikidata":"https://www.wikidata.org/wiki/Q1570441","display_name":"Parameterized complexity","level":2,"score":0.3797000050544739},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.320499986410141},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.30489999055862427},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.301800012588501},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.3009999990463257},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.2980000078678131},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.29159998893737793},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.267300009727478}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.25488","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.25488","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.25488","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.25488","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Knowledge":[0],"Distillation":[1],"(KD)":[2],"is":[3,25,137,145,222],"widely":[4],"used":[5],"to":[6,79,160,162,170,213,265],"obtain":[7],"compact":[8],"models":[9],"for":[10,57],"efficient":[11,98,124,225],"inference":[12],"in":[13,87,129,192],"resource-constrained":[14],"environments.":[15],"Yet":[16],"the":[17,21,29,158,188,193,206,210,230,267,273],"computational":[18],"overhead":[19],"of":[20,31,126,232,251,269],"distillation":[22,215],"process":[23],"itself":[24],"often":[26,85],"overlooked,":[27],"raising":[28],"question":[30],"whether":[32],"a":[33,107,134,149,184,198,248],"better":[34],"student":[35],"model":[36],"can":[37,118],"be":[38],"obtained":[39],"with":[40,106,197,229],"less":[41,44,260,263],"data":[42,47,51,99,261],"and":[43,123,226,238,262],"compute":[45,264],"via":[46],"pruning.":[48],"However,":[49],"existing":[50],"pruning":[52,100,164,178,252],"methods":[53],"are":[54],"not":[55],"designed":[56],"KD:":[58],"some":[59],"introduce":[60],"substantial":[61],"overhead,":[62],"such":[63],"as":[64,120,179],"obtaining":[65],"training":[66],"dynamics":[67],"through":[68,183],"retraining,":[69],"while":[70,205],"others":[71],"rely":[72],"on":[73,140,236,259,272],"heuristic":[74,172],"selection":[75],"rules":[76],"that":[77,102,115,220,241],"fail":[78],"capture":[80],"what":[81],"KD":[82,130,177],"actually":[83],"requires,":[84],"resulting":[86],"suboptimal":[88],"subsets.":[89],"To":[90],"address":[91],"these":[92],"issues,":[93],"we":[94,112,175],"propose":[95],"IF-Beta,":[96],"an":[97,121],"framework":[101],"combines":[103],"influence":[104,116],"functions":[105,117],"learnable":[108],"sampling":[109,143],"policy.":[110],"Empirically,":[111],"first":[113],"demonstrate":[114],"serve":[119],"effective":[122],"estimator":[125],"sample":[127],"impact":[128],"settings,":[131],"where":[132,187],"only":[133],"pretrained":[135],"teacher":[136,194],"available.":[138],"Building":[139],"this,":[141],"our":[142],"policy":[144,159,182,211],"specifically":[146],"parameterized":[147],"by":[148],"Beta":[150],"distribution,":[151],"whose":[152],"highly":[153],"flexible":[154],"two-parameter":[155],"family":[156],"allows":[157],"adapt":[161],"diverse":[163],"regimes":[165],"rather":[166],"than":[167],"being":[168],"tied":[169],"fixed":[171],"forms.":[173],"Next,":[174],"formulate":[176],"optimizing":[180],"this":[181],"bilevel":[185],"objective,":[186,200],"inner":[189],"loop":[190,208],"operates":[191],"feature":[195],"space":[196],"KD-aligned":[199],"enabling":[201],"fast":[202],"proxy":[203],"training,":[204],"outer":[207],"updates":[209],"parameters":[212],"maximize":[214],"performance.":[216],"This":[217],"design":[218],"ensures":[219],"IF-Beta":[221,242,255],"both":[223],"computationally":[224],"inherently":[227],"aligned":[228],"goals":[231],"KD.":[233],"Extensive":[234],"experiments":[235],"CIFAR-10/100":[237],"ImageNet":[239],"show":[240],"consistently":[243],"outperforms":[244],"other":[245],"baselines":[246],"across":[247],"wide":[249],"range":[250],"ratios.":[253],"Remarkably,":[254],"enables":[256],"students":[257,270],"trained":[258],"surpass":[266],"performance":[268],"distilled":[271],"full":[274],"dataset.":[275]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-26T00:00:00"}