{"id":"https://openalex.org/W3119460773","doi":"https://doi.org/10.1007/s40747-020-00248-y","title":"Knowledge from the original network: restore a better pruned network with knowledge distillation","display_name":"Knowledge from the original network: restore a better pruned network with knowledge distillation","publication_year":2021,"publication_date":"2021-01-10","ids":{"openalex":"https://openalex.org/W3119460773","doi":"https://doi.org/10.1007/s40747-020-00248-y","mag":"3119460773"},"language":"en","primary_location":{"id":"doi:10.1007/s40747-020-00248-y","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40747-020-00248-y","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40747-020-00248-y.pdf","source":{"id":"https://openalex.org/S3035462843","display_name":"Complex & Intelligent Systems","issn_l":"2198-6053","issn":["2198-6053","2199-4536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Complex & Intelligent Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://link.springer.com/content/pdf/10.1007/s40747-020-00248-y.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101849127","display_name":"Liyang Chen","orcid":"https://orcid.org/0000-0001-6985-8281"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liyang Chen","raw_affiliation_strings":["School of Mechanical Engineering, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Mechanical Engineering, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102007341","display_name":"Yongquan Chen","orcid":"https://orcid.org/0000-0003-2493-6033"},"institutions":[{"id":"https://openalex.org/I4210104064","display_name":"Shenzhen Academy of Robotics","ror":"https://ror.org/01h027j09","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210104064"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yongquan Chen","raw_affiliation_strings":["Shenzhen Institue of Artificial Intelligence and Robotics for Society(AIRS), Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shenzhen Institue of Artificial Intelligence and Robotics for Society(AIRS), Shenzhen, China","institution_ids":["https://openalex.org/I4210104064"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050259999","display_name":"Juntong Xi","orcid":"https://orcid.org/0000-0001-7701-0010"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Juntong Xi","raw_affiliation_strings":["School of Mechanical Engineering, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Mechanical Engineering, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052481491","display_name":"Xinyi Le","orcid":"https://orcid.org/0000-0003-0318-9497"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4210104064","display_name":"Shenzhen Academy of Robotics","ror":"https://ror.org/01h027j09","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210104064"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xinyi Le","raw_affiliation_strings":["Department of Automation, Shanghai Jiao Tong University, Shanghai, China","Shenzhen Institue of Artificial Intelligence and Robotics for Society(AIRS), Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0003-0318-9497","affiliations":[{"raw_affiliation_string":"Department of Automation, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Shenzhen Institue of Artificial Intelligence and Robotics for Society(AIRS), Shenzhen, China","institution_ids":["https://openalex.org/I4210104064"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5052481491"],"corresponding_institution_ids":["https://openalex.org/I183067930","https://openalex.org/I4210104064"],"apc_list":{"value":1320,"currency":"GBP","value_usd":1619},"apc_paid":{"value":1320,"currency":"GBP","value_usd":1619},"fwci":4.8994,"has_fulltext":true,"cited_by_count":45,"citation_normalized_percentile":{"value":0.95794442,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"8","issue":"2","first_page":"709","last_page":"718"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12814","display_name":"Gaussian Processes and Bayesian Inference","score":0.986299991607666,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7178503274917603},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.6205638647079468},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5796109437942505},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5746710300445557},{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.5340384840965271},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4791771173477173},{"id":"https://openalex.org/keywords/network-architecture","display_name":"Network architecture","score":0.4349218010902405}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7178503274917603},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6205638647079468},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5796109437942505},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5746710300445557},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.5340384840965271},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4791771173477173},{"id":"https://openalex.org/C193415008","wikidata":"https://www.wikidata.org/wiki/Q639681","display_name":"Network architecture","level":2,"score":0.4349218010902405},{"id":"https://openalex.org/C6557445","wikidata":"https://www.wikidata.org/wiki/Q173113","display_name":"Agronomy","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s40747-020-00248-y","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40747-020-00248-y","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40747-020-00248-y.pdf","source":{"id":"https://openalex.org/S3035462843","display_name":"Complex & Intelligent Systems","issn_l":"2198-6053","issn":["2198-6053","2199-4536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Complex & Intelligent Systems","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s40747-020-00248-y","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40747-020-00248-y","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40747-020-00248-y.pdf","source":{"id":"https://openalex.org/S3035462843","display_name":"Complex & Intelligent Systems","issn_l":"2198-6053","issn":["2198-6053","2199-4536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Complex & Intelligent Systems","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6499999761581421,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G3096721518","display_name":null,"funder_award_id":"2019ZE057001","funder_id":"https://openalex.org/F4320322857","funder_display_name":"Aeronautical Science Foundation of China"},{"id":"https://openalex.org/G870084144","display_name":null,"funder_award_id":"20QC1401100","funder_id":"https://openalex.org/F4320327803","funder_display_name":"Shanghai Rising-Star Program"}],"funders":[{"id":"https://openalex.org/F4320322857","display_name":"Aeronautical Science Foundation of China","ror":"https://ror.org/02wq41p38"},{"id":"https://openalex.org/F4320327803","display_name":"Shanghai Rising-Star Program","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3119460773.pdf","grobid_xml":"https://content.openalex.org/works/W3119460773.grobid-xml"},"referenced_works_count":43,"referenced_works":["https://openalex.org/W174941419","https://openalex.org/W569478347","https://openalex.org/W1524097605","https://openalex.org/W1821462560","https://openalex.org/W2114766824","https://openalex.org/W2161591461","https://openalex.org/W2184652140","https://openalex.org/W2194775991","https://openalex.org/W2302255633","https://openalex.org/W2473930607","https://openalex.org/W2524428287","https://openalex.org/W2561238782","https://openalex.org/W2764043458","https://openalex.org/W2769215776","https://openalex.org/W2799404913","https://openalex.org/W2803023299","https://openalex.org/W2805003733","https://openalex.org/W2899565453","https://openalex.org/W2904206140","https://openalex.org/W2962835968","https://openalex.org/W2962965870","https://openalex.org/W2963140444","https://openalex.org/W2963518130","https://openalex.org/W2963534679","https://openalex.org/W2963674932","https://openalex.org/W2963981420","https://openalex.org/W2964137095","https://openalex.org/W2964299589","https://openalex.org/W2969601108","https://openalex.org/W2981794819","https://openalex.org/W2982157312","https://openalex.org/W2982242214","https://openalex.org/W2995105044","https://openalex.org/W2995505408","https://openalex.org/W2996569511","https://openalex.org/W2997006708","https://openalex.org/W3002332355","https://openalex.org/W3011537656","https://openalex.org/W3034368386","https://openalex.org/W3034584726","https://openalex.org/W3039537481","https://openalex.org/W4210643868","https://openalex.org/W6601865935"],"related_works":["https://openalex.org/W2373300491","https://openalex.org/W2378744544","https://openalex.org/W2594301978","https://openalex.org/W2379704676","https://openalex.org/W4206442282","https://openalex.org/W2384505857","https://openalex.org/W2355171581","https://openalex.org/W2145253956","https://openalex.org/W4229439743","https://openalex.org/W4205470293"],"abstract_inverted_index":{"Abstract":[0],"To":[1],"deploy":[2],"deep":[3,23,44,178],"neural":[4,45,180],"networks":[5,227],"to":[6,34,60,83,95,117,126,160],"edge":[7],"devices":[8],"with":[9,165,176,192,204,228],"limited":[10],"computation":[11],"and":[12,80,142,156],"storage":[13],"costs,":[14],"model":[15,31,39],"compression":[16],"is":[17,47,65,77],"necessary":[18],"for":[19],"the":[20,36,49,52,62,73,91,97,107,113,119,123,127,158,161,173,185,216,220,232],"application":[21],"of":[22,30,38,51,100,106],"learning.":[24],"Pruning,":[25],"as":[26,112,219,231],"a":[27,43,151],"traditional":[28,58,162],"way":[29,59],"compression,":[32],"seeks":[33],"reduce":[35],"parameters":[37,70],"weights.":[40],"However,":[41],"when":[42],"network":[46,53,105,109,125,187,195,218],"pruned,":[48,72],"accuracy":[50,63,98,198,230],"will":[54],"significantly":[55],"decrease.":[56],"The":[57,103],"decrease":[61],"loss":[64,99],"fine-tuning.":[66],"When":[67],"over":[68],"many":[69],"are":[71],"pruned":[74,101,108,128,186],"network\u2019s":[75],"capacity":[76],"reduced":[78],"heavily":[79],"cannot":[81],"recover":[82],"high":[84],"accuracy.":[85],"In":[86],"this":[87,213],"paper,":[88],"we":[89],"apply":[90],"knowledge":[92,121,135,144,190],"distillation":[93,136,191],"strategy":[94],"abate":[96],"models.":[102],"original":[104,124,194,217],"was":[110],"used":[111],"teacher":[114,221],"network,":[115],"aiming":[116],"transfer":[118],"dark":[120],"from":[122],"sub-network.":[129],"We":[130],"have":[131,169],"applied":[132],"three":[133],"mainstream":[134],"methods:":[137],"response-based":[138],"knowledge,":[139,141],"feature-based":[140],"relation-based":[143],"(Gou":[145],"et":[146],"al.":[147],"in":[148,212],"Knowledge":[149],"distillation:":[150],"survey.":[152],"arXiv:200605525":[153],",":[154],"2020),":[155],"compare":[157],"result":[159],"fine-tuning":[163,203],"method":[164],"grand-truth":[166],"labels.":[167,206],"Experiments":[168],"been":[170,210],"done":[171],"on":[172],"CIFAR100":[174],"dataset":[175],"several":[177],"convolution":[179],"network.":[181],"Results":[182],"show":[183],"that":[184,215],"recovered":[188,201],"by":[189,202],"its":[193],"performs":[196,222],"better":[197,223],"than":[199,224],"it":[200],"sample":[205],"It":[207],"has":[208],"also":[209],"validated":[211],"paper":[214],"differently":[225],"structured":[226],"same":[229],"teacher.":[233]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":14},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":6}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}