{"id":"https://openalex.org/W4414084161","doi":"https://doi.org/10.1109/tpami.2025.3607982","title":"Toward Effective Knowledge Distillation: Navigating Beyond Small-Data Pitfall","display_name":"Toward Effective Knowledge Distillation: Navigating Beyond Small-Data Pitfall","publication_year":2025,"publication_date":"2025-09-09","ids":{"openalex":"https://openalex.org/W4414084161","doi":"https://doi.org/10.1109/tpami.2025.3607982","pmid":"https://pubmed.ncbi.nlm.nih.gov/40924515"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2025.3607982","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3607982","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Zhiwei Hao","orcid":"https://orcid.org/0000-0002-6237-7028"},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiwei Hao","raw_affiliation_strings":["School of Information and Electrionics, Beijing Institute of Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-6237-7028","affiliations":[{"raw_affiliation_string":"School of Information and Electrionics, Beijing Institute of Technology, Beijing, China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016185815","display_name":"Jianyuan Guo","orcid":"https://orcid.org/0000-0002-2981-1953"},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]},{"id":"https://openalex.org/I168719708","display_name":"City University of Hong Kong","ror":"https://ror.org/03q8dnn23","country_code":"HK","type":"education","lineage":["https://openalex.org/I168719708"]}],"countries":["AU","HK"],"is_corresponding":false,"raw_author_name":"Jianyuan Guo","raw_affiliation_strings":["Department of Computer Science, City University of Hong Kong, Hong Kong, SAR, China","School of Computer Science, Faculty of Engineering, University of Sydney, 6 Cleveland Street, Darlington, NSW, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, City University of Hong Kong, Hong Kong, SAR, China","institution_ids":["https://openalex.org/I168719708"]},{"raw_affiliation_string":"School of Computer Science, Faculty of Engineering, University of Sydney, 6 Cleveland Street, Darlington, NSW, Australia","institution_ids":["https://openalex.org/I129604602"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Kai Han","orcid":"https://orcid.org/0000-0002-9761-2702"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]},{"id":"https://openalex.org/I4210159102","display_name":"Huawei Technologies (Sweden)","ror":"https://ror.org/0500fyd17","country_code":"SE","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210159102"]}],"countries":["CN","SE"],"is_corresponding":false,"raw_author_name":"Kai Han","raw_affiliation_strings":["Huawei Noah’s Ark Lab, Beijing, China","Huawei Noah's Ark Lab, HuaWei Building, No.3 Xinxi Road, ShangDi Information Industry Base, Hai-Dian District, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9761-2702","affiliations":[{"raw_affiliation_string":"Huawei Noah’s Ark Lab, Beijing, China","institution_ids":["https://openalex.org/I2250955327"]},{"raw_affiliation_string":"Huawei Noah's Ark Lab, HuaWei Building, No.3 Xinxi Road, ShangDi Information Industry Base, Hai-Dian District, Beijing, China","institution_ids":["https://openalex.org/I2250955327","https://openalex.org/I4210159102"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Han Hu","orcid":"https://orcid.org/0000-0001-7532-0496"},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Han Hu","raw_affiliation_strings":["School of Information and Electrionics, Beijing Institute of Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-7532-0496","affiliations":[{"raw_affiliation_string":"School of Information and Electrionics, Beijing Institute of Technology, Beijing, China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001529504","display_name":"Chang Xu","orcid":"https://orcid.org/0000-0002-4756-0609"},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Chang Xu","raw_affiliation_strings":["School of Computer Science, Faculty of Engineering, University of Sydney, Sydney, NSW, Australia","School of Computer Science, Faculty of Engineering, University of Sydney, 6 Cleveland Street, Darlington, NSW, Australia"],"raw_orcid":"https://orcid.org/0000-0002-4756-0609","affiliations":[{"raw_affiliation_string":"School of Computer Science, Faculty of Engineering, University of Sydney, Sydney, NSW, Australia","institution_ids":["https://openalex.org/I129604602"]},{"raw_affiliation_string":"School of Computer Science, Faculty of Engineering, University of Sydney, 6 Cleveland Street, Darlington, NSW, Australia","institution_ids":["https://openalex.org/I129604602"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5059936656","display_name":"Yunhe Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]},{"id":"https://openalex.org/I4210159102","display_name":"Huawei Technologies (Sweden)","ror":"https://ror.org/0500fyd17","country_code":"SE","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210159102"]}],"countries":["CN","SE"],"is_corresponding":false,"raw_author_name":"Yunhe Wang","raw_affiliation_strings":["Huawei Noah’s Ark Lab, Beijing, China","Huawei Noah's Ark Lab, HuaWei Building, No.3 Xinxi Road, ShangDi Information Industry Base, Hai-Dian District, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-2709-4946","affiliations":[{"raw_affiliation_string":"Huawei Noah’s Ark Lab, Beijing, China","institution_ids":["https://openalex.org/I2250955327"]},{"raw_affiliation_string":"Huawei Noah's Ark Lab, HuaWei Building, No.3 Xinxi Road, ShangDi Information Industry Base, Hai-Dian District, Beijing, China","institution_ids":["https://openalex.org/I2250955327","https://openalex.org/I4210159102"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.11956604,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"48","issue":"1","first_page":"542","last_page":"556"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9929999709129333,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9929999709129333,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12611","display_name":"Neural Networks and Reservoir Computing","score":0.9790999889373779,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9514999985694885,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.44290000200271606},{"id":"https://openalex.org/keywords/enhanced-data-rates-for-gsm-evolution","display_name":"Enhanced Data Rates for GSM Evolution","score":0.4242999851703644},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.38580000400543213},{"id":"https://openalex.org/keywords/knowledge-transfer","display_name":"Knowledge transfer","score":0.3441999852657318},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.3434000015258789},{"id":"https://openalex.org/keywords/transfer-of-learning","display_name":"Transfer of learning","score":0.32679998874664307}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7243000268936157},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5141000151634216},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4916999936103821},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.44290000200271606},{"id":"https://openalex.org/C162307627","wikidata":"https://www.wikidata.org/wiki/Q204833","display_name":"Enhanced Data Rates for GSM Evolution","level":2,"score":0.4242999851703644},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.38580000400543213},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38100001215934753},{"id":"https://openalex.org/C2776960227","wikidata":"https://www.wikidata.org/wiki/Q2586354","display_name":"Knowledge transfer","level":2,"score":0.3441999852657318},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3434000015258789},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.32679998874664307},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.3188000023365021},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.3125999867916107},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.25130000710487366}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2025.3607982","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3607982","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:40924515","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40924515","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5019704030","display_name":null,"funder_award_id":"U2336211","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W153185079","https://openalex.org/W1861492603","https://openalex.org/W2031489346","https://openalex.org/W2108598243","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2331143823","https://openalex.org/W2340897893","https://openalex.org/W2531409750","https://openalex.org/W2620998106","https://openalex.org/W2739879705","https://openalex.org/W2743289088","https://openalex.org/W2884822772","https://openalex.org/W2896457183","https://openalex.org/W2949736877","https://openalex.org/W2963140444","https://openalex.org/W2983943451","https://openalex.org/W2986015886","https://openalex.org/W2987861506","https://openalex.org/W2992308087","https://openalex.org/W2997006708","https://openalex.org/W3004127093","https://openalex.org/W3034756453","https://openalex.org/W3035160371","https://openalex.org/W3035321581","https://openalex.org/W3035682985","https://openalex.org/W3094502228","https://openalex.org/W3108124733","https://openalex.org/W3138516171","https://openalex.org/W3170224286","https://openalex.org/W3176459575","https://openalex.org/W3177196641","https://openalex.org/W4214524539","https://openalex.org/W4214634256","https://openalex.org/W4226426325","https://openalex.org/W4246193833","https://openalex.org/W4281489207","https://openalex.org/W4312443924","https://openalex.org/W4312677967","https://openalex.org/W4312929415","https://openalex.org/W4312980183","https://openalex.org/W4385245566","https://openalex.org/W4386076493","https://openalex.org/W4394698525","https://openalex.org/W4402660160"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"The":[0],"spectacular":[1],"success":[2],"of":[3,13,61,85,104,118,150],"training":[4],"large":[5],"models":[6,22,45],"on":[7,23,97],"extensive":[8],"datasets":[9,47],"highlights":[10],"the":[11,59,83,102,119,168,190,195],"potential":[12],"scaling":[14],"up":[15,50],"for":[16,67,133],"exceptional":[17],"performance.":[18],"To":[19,100],"deploy":[20],"these":[21],"edge":[24],"devices,":[25],"knowledge":[26,120],"distillation":[27],"(KD)":[28],"is":[29,55,131],"commonly":[30],"used":[31],"to":[32,57,92,188],"create":[33],"a":[34,38,86,115,154],"compact":[35],"model":[36],"from":[37],"larger,":[39],"pretrained":[40],"teacher":[41],"model.":[42],"However,":[43],"as":[44],"and":[46,70,81,198],"rapidly":[48],"scale":[49],"in":[51,141,192],"practical":[52],"applications,":[53],"it":[54],"crucial":[56,132],"consider":[58],"applicability":[60],"existing":[62],"KD":[63,79,94,107,137,160,178],"approaches":[64],"originally":[65],"designed":[66],"limited-capacity":[68],"architectures":[69],"small-scale":[71],"datasets.":[72,99],"In":[73,148],"this":[74],"paper,":[75],"we":[76,113,152],"revisit":[77],"current":[78],"methods":[80,108],"identify":[82],"presence":[84],"small-data":[87,196],"pitfall,":[88],"where":[89],"most":[90],"modifications":[91,140],"vanilla":[93,159],"prove":[95],"ineffective":[96],"large-scale":[98],"guide":[101,189],"design":[103],"consistently":[105,135,200],"effective":[106,136,201],"across":[109],"different":[110],"data":[111],"scales,":[112],"conduct":[114],"meticulous":[116],"evaluation":[117],"transfer":[121],"process.":[122],"Our":[123],"findings":[124],"reveal":[125],"that":[126,157],"incorporating":[127,164],"more":[128],"useful":[129],"information":[130,166],"achieving":[134],"methods,":[138],"while":[139],"loss":[142],"functions":[143],"show":[144],"relatively":[145],"less":[146],"significance.":[147],"light":[149],"this,":[151],"present":[153],"paradigmatic":[155],"example":[156],"combines":[158],"with":[161],"deep":[162],"supervision,":[163],"additional":[165],"into":[167],"student":[169],"during":[170],"distillation.":[171],"This":[172],"approach":[173],"surpasses":[174],"almost":[175],"all":[176],"recent":[177],"methods.":[179],"We":[180],"believe":[181],"our":[182],"study":[183],"will":[184],"offer":[185],"valuable":[186],"insights":[187],"community":[191],"navigating":[193],"beyond":[194],"pitfall":[197],"toward":[199],"KD.":[202]},"counts_by_year":[],"updated_date":"2025-12-21T01:58:51.020947","created_date":"2025-10-10T00:00:00"}