{"id":"https://openalex.org/W4387385613","doi":"https://doi.org/10.1109/tmm.2023.3321480","title":"Parameter-Efficient and Student-Friendly Knowledge Distillation","display_name":"Parameter-Efficient and Student-Friendly Knowledge Distillation","publication_year":2023,"publication_date":"2023-10-05","ids":{"openalex":"https://openalex.org/W4387385613","doi":"https://doi.org/10.1109/tmm.2023.3321480"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2023.3321480","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2023.3321480","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053687590","display_name":"Jun Rao","orcid":"https://orcid.org/0000-0001-5804-1508"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]},{"id":"https://openalex.org/I4210103986","display_name":"Jingdong (China)","ror":"https://ror.org/01dkjkq64","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210103986"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Rao","raw_affiliation_strings":["Harbin Institute of Technology, Shenzhen, China","JD Explore Academy, China"],"raw_orcid":"https://orcid.org/0000-0001-5804-1508","affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]},{"raw_affiliation_string":"JD Explore Academy, China","institution_ids":["https://openalex.org/I4210103986"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063175937","display_name":"Xv Meng","orcid":"https://orcid.org/0009-0005-4631-8203"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xv Meng","raw_affiliation_strings":["Harbin Institute of Technology, Shenzhen, China"],"raw_orcid":"https://orcid.org/0009-0005-4631-8203","affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100456723","display_name":"Liang Ding","orcid":"https://orcid.org/0000-0001-8976-2084"},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Liang Ding","raw_affiliation_strings":["the School of Computer Science, University of Sydney, NSW, Australia"],"raw_orcid":"https://orcid.org/0000-0001-8976-2084","affiliations":[{"raw_affiliation_string":"the School of Computer Science, University of Sydney, NSW, Australia","institution_ids":["https://openalex.org/I129604602"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050911130","display_name":"Shuhan Qi","orcid":"https://orcid.org/0000-0002-6903-145X"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]},{"id":"https://openalex.org/I4210136793","display_name":"Peng Cheng Laboratory","ror":"https://ror.org/03qdqbt06","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210136793"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuhan Qi","raw_affiliation_strings":["the Harbin Institute of Technology, Shenzhen, China","Guangdong Provincial Key Laboratory of Novel Security Intelligence Technologies, Shenzhen, China","Peng Cheng Laboratory, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-6903-145X","affiliations":[{"raw_affiliation_string":"the Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]},{"raw_affiliation_string":"Guangdong Provincial Key Laboratory of Novel Security Intelligence Technologies, Shenzhen, China","institution_ids":[]},{"raw_affiliation_string":"Peng Cheng Laboratory, Shenzhen, China","institution_ids":["https://openalex.org/I4210136793"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101599729","display_name":"Xuebo Liu","orcid":"https://orcid.org/0000-0001-8524-2006"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuebo Liu","raw_affiliation_strings":["Harbin Institute of Technology, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0001-8524-2006","affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100402911","display_name":"Min Zhang","orcid":"https://orcid.org/0000-0002-3895-5510"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Min Zhang","raw_affiliation_strings":["Harbin Institute of Technology, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-3895-5510","affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074103823","display_name":"Dacheng Tao","orcid":"https://orcid.org/0000-0001-7225-5449"},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Dacheng Tao","raw_affiliation_strings":["the School of Computer Science, University of Sydney, NSW, Australia"],"raw_orcid":"https://orcid.org/0000-0001-7225-5449","affiliations":[{"raw_affiliation_string":"the School of Computer Science, University of Sydney, NSW, Australia","institution_ids":["https://openalex.org/I129604602"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":9.0417,"has_fulltext":false,"cited_by_count":57,"citation_normalized_percentile":{"value":0.9837642,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"26","issue":null,"first_page":"4230","last_page":"4241"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7980347871780396},{"id":"https://openalex.org/keywords/adapter","display_name":"Adapter (computing)","score":0.6207107305526733},{"id":"https://openalex.org/keywords/knowledge-transfer","display_name":"Knowledge transfer","score":0.5439098477363586},{"id":"https://openalex.org/keywords/smoothing","display_name":"Smoothing","score":0.5385720729827881},{"id":"https://openalex.org/keywords/smoothness","display_name":"Smoothness","score":0.5108170509338379},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5054478645324707},{"id":"https://openalex.org/keywords/transfer-of-learning","display_name":"Transfer of learning","score":0.4968879520893097},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.47654110193252563},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.47186869382858276},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.45709365606307983},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.11256459355354309},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09737411141395569}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7980347871780396},{"id":"https://openalex.org/C177284502","wikidata":"https://www.wikidata.org/wiki/Q1005390","display_name":"Adapter (computing)","level":2,"score":0.6207107305526733},{"id":"https://openalex.org/C2776960227","wikidata":"https://www.wikidata.org/wiki/Q2586354","display_name":"Knowledge transfer","level":2,"score":0.5439098477363586},{"id":"https://openalex.org/C3770464","wikidata":"https://www.wikidata.org/wiki/Q775963","display_name":"Smoothing","level":2,"score":0.5385720729827881},{"id":"https://openalex.org/C102634674","wikidata":"https://www.wikidata.org/wiki/Q868473","display_name":"Smoothness","level":2,"score":0.5108170509338379},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5054478645324707},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.4968879520893097},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47654110193252563},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.47186869382858276},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.45709365606307983},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.11256459355354309},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09737411141395569},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2023.3321480","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2023.3321480","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7099999785423279}],"awards":[{"id":"https://openalex.org/G5421374829","display_name":null,"funder_award_id":"62372139","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":51,"referenced_works":["https://openalex.org/W2108598243","https://openalex.org/W2183341477","https://openalex.org/W2187089797","https://openalex.org/W2620998106","https://openalex.org/W2887783173","https://openalex.org/W2923014074","https://openalex.org/W2963140444","https://openalex.org/W2964111476","https://openalex.org/W2970454332","https://openalex.org/W2981819252","https://openalex.org/W2986015886","https://openalex.org/W2987809065","https://openalex.org/W2996970889","https://openalex.org/W2997006708","https://openalex.org/W3004127093","https://openalex.org/W3034368386","https://openalex.org/W3035321581","https://openalex.org/W3101248447","https://openalex.org/W3105966348","https://openalex.org/W3118608800","https://openalex.org/W3138154797","https://openalex.org/W3176055902","https://openalex.org/W3177196641","https://openalex.org/W3177323791","https://openalex.org/W3197901717","https://openalex.org/W3204703845","https://openalex.org/W3209108016","https://openalex.org/W4226101686","https://openalex.org/W4281571140","https://openalex.org/W4284698859","https://openalex.org/W4285202066","https://openalex.org/W4292948773","https://openalex.org/W4361230739","https://openalex.org/W4365801687","https://openalex.org/W4385573520","https://openalex.org/W6638523607","https://openalex.org/W6691459498","https://openalex.org/W6748082341","https://openalex.org/W6752186649","https://openalex.org/W6759579507","https://openalex.org/W6761472960","https://openalex.org/W6769906912","https://openalex.org/W6787972765","https://openalex.org/W6790545024","https://openalex.org/W6796581206","https://openalex.org/W6796782416","https://openalex.org/W6802744804","https://openalex.org/W6804620367","https://openalex.org/W6839568498","https://openalex.org/W6841870936","https://openalex.org/W6851629679"],"related_works":["https://openalex.org/W2133028525","https://openalex.org/W4229060448","https://openalex.org/W4306381730","https://openalex.org/W2981692913","https://openalex.org/W3044188621","https://openalex.org/W3184035966","https://openalex.org/W2485605994","https://openalex.org/W2160602540","https://openalex.org/W4385571108","https://openalex.org/W4200251711"],"abstract_inverted_index":{"Pre-trained":[0],"models":[1,10],"are":[2,53,129],"frequently":[3],"employed":[4],"in":[5,68],"multimodal":[6],"learning.":[7],"However,":[8],"these":[9],"have":[11,104],"too":[12,17],"many":[13],"parameters":[14,49],"and":[15,88,133,143,147,180,210,228],"need":[16],"much":[18],"effort":[19],"to":[20,31,43,72,95,118,215,244],"fine-tune":[21],"the":[22,35,48,51,73,79,97,112,120,130,136,139,151,164,172,175,194,198,208,213,236],"downstream":[23],"tasks.":[24],"Knowledge":[25],"distillation":[26,247],"(KD)":[27],"is":[28],"a":[29,44,101,144,188],"method":[30],"transfer":[32,70,149],"knowledge":[33,69,154],"using":[34],"soft":[36,98,116,141,161,199,217],"label":[37,86,99,142],"from":[38],"this":[39,63],"pre-trained":[40,152],"teacher":[41,52,102,165,209],"model":[42,75],"smaller":[45],"student,":[46],"where":[47,184],"of":[50,83,100,114,126,135,138,150,163,197],"fixed":[54],"(or":[55],"partially)":[56],"during":[57],"training.":[58],"Recent":[59],"studies":[60],"show":[61,186,230],"that":[62,231],"mode":[64],"may":[65],"cause":[66],"difficulties":[67],"due":[71],"mismatched":[74],"capacities.":[76],"To":[77],"alleviate":[78],"mismatch":[80,173],"problem,":[81],"adjustment":[82],"temperature":[84],"parameters,":[85],"smoothing":[87],"teacher-student":[89],"joint":[90],"training":[91,237],"methods":[92,109],"(online":[93],"distillation)":[94],"smooth":[96],"network,":[103],"been":[105],"proposed.":[106],"But":[107],"those":[108],"rarely":[110],"explain":[111],"effect":[113,137],"smoothed":[115,140],"labels":[117,162,218],"enhance":[119],"KD":[121],"performance.":[122],"The":[123],"main":[124],"contributions":[125],"our":[127],"work":[128],"discovery,":[131],"analysis,":[132],"validation":[134],"less":[145],"time-consuming":[146],"adaptive":[148,159],"teacher's":[153,179],"method,":[155],"namely":[156],"PESF-KD":[157,232],"by":[158],"tuning":[160],"network.":[166],"Technically,":[167],"we":[168,185,202],"first":[169],"mathematically":[170],"formulate":[171],"as":[174],"sharpness":[176],"gap":[177,189],"between":[178],"student's":[181],"predictive":[182],"distributions,":[183],"such":[187],"can":[190,233],"be":[191],"narrowed":[192],"with":[193,219],"appropriate":[195,220],"smoothness":[196],"label.":[200],"Then,":[201],"introduce":[203],"an":[204],"adapter":[205,214],"module":[206],"for":[207],"only":[211],"update":[212],"obtain":[216],"smoothness.":[221],"Experiments":[222],"on":[223],"various":[224],"benchmarks":[225],"including":[226],"CV":[227],"NLP":[229],"significantly":[234],"reduce":[235],"cost":[238],"while":[239],"obtaining":[240],"competitive":[241],"results":[242],"compared":[243],"advanced":[245],"online":[246],"methods.":[248]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":19},{"year":2024,"cited_by_count":22},{"year":2023,"cited_by_count":8}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}