{"id":"https://openalex.org/W3139941584","doi":"https://doi.org/10.48550/arxiv.2103.16844","title":"Fixing the Teacher-Student Knowledge Discrepancy in Distillation","display_name":"Fixing the Teacher-Student Knowledge Discrepancy in Distillation","publication_year":2021,"publication_date":"2021-03-31","ids":{"openalex":"https://openalex.org/W3139941584","doi":"https://doi.org/10.48550/arxiv.2103.16844","mag":"3139941584"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2103.16844","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2103.16844","pdf_url":"https://arxiv.org/pdf/2103.16844","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2103.16844","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5112484491","display_name":"Jiangfan Han","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Jiangfan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085272181","display_name":"Mengya Gao","orcid":"https://orcid.org/0000-0003-2736-6005","observed_orcids":["https://orcid.org/0000-0003-2736-6005"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Mengya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081141309","display_name":"Yujie Wang","orcid":"https://orcid.org/0000-0002-3225-609X","observed_orcids":["https://orcid.org/0000-0002-3225-609X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yujie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101644716","display_name":"Quanquan Li","orcid":"https://orcid.org/0000-0002-8518-2346","observed_orcids":["https://orcid.org/0000-0002-8518-2346"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Quanquan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100732450","display_name":"Hongsheng Li","orcid":"https://orcid.org/0000-0002-2664-7975","observed_orcids":["https://orcid.org/0000-0002-2664-7975"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Hongsheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5100444820","display_name":"Xiaogang Wang","orcid":"https://orcid.org/0000-0002-7929-5889","observed_orcids":["https://orcid.org/0000-0002-7929-5889"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xiaogang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9365000128746033,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9365000128746033,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.06199999898672104,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.0008999999845400453,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/knowledge-distillation","display_name":"knowledge distillation","score":0.9909999966621399},{"id":"https://openalex.org/keywords/image-classification","display_name":"image classification","score":0.6349999904632568},{"id":"https://openalex.org/keywords/object-detection","display_name":"object detection","score":1.0}],"concepts":[{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.6235132217407227},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.5357320308685303},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.38406985998153687},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.36187443137168884},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3439328372478485},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.2734578549861908},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.258465051651001}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2103.16844","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2103.16844","pdf_url":"https://arxiv.org/pdf/2103.16844","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2103.16844","doi":"https://doi.org/10.48550/arxiv.2103.16844","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2103.16844","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2103.16844","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2103.16844","pdf_url":"https://arxiv.org/pdf/2103.16844","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.975600004196167,"display_name":"Quality education"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W1690739335","https://openalex.org/W1821462560","https://openalex.org/W1861492603","https://openalex.org/W2108598243","https://openalex.org/W2134797427","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2561238782","https://openalex.org/W2565639579","https://openalex.org/W2626778328","https://openalex.org/W2739879705","https://openalex.org/W2805003733","https://openalex.org/W2886756692","https://openalex.org/W2919347051","https://openalex.org/W2921853515","https://openalex.org/W2948635472","https://openalex.org/W2955192706","https://openalex.org/W2963140444","https://openalex.org/W2963351448","https://openalex.org/W2963446712","https://openalex.org/W2964111476","https://openalex.org/W2964161024","https://openalex.org/W2970866842","https://openalex.org/W2981819252","https://openalex.org/W2982157312","https://openalex.org/W3118608800"],"related_works":["https://openalex.org/W3169549154","https://openalex.org/W4392739013","https://openalex.org/W7221767288","https://openalex.org/W3204703845","https://openalex.org/W3199022179","https://openalex.org/W3211605522","https://openalex.org/W2979125527","https://openalex.org/W4282972970","https://openalex.org/W6977098252","https://openalex.org/W4386852221"],"abstract_inverted_index":{"Training":[0],"a":[1,9,125],"small":[2],"student":[3,42,55,104,111,141,150],"network":[4,12],"with":[5,57,139,197],"the":[6,19,22,25,28,74,93,97,108,140,144,169,181],"guidance":[7],"of":[8,21,86,101,110,183],"larger":[10],"teacher":[11,40,53,102,113],"is":[13,34,189],"an":[14],"effective":[15],"way":[16],"to":[17,32,148],"promote":[18],"performance":[20],"student.":[23],"Despite":[24],"different":[26,39,58,63,71,79,149,157],"types,":[27],"guided":[29],"knowledge":[30,46,90,98,130,136,147,172],"used":[31],"distill":[33],"always":[35],"kept":[36],"unchanged":[37],"for":[38,78,152],"and":[41,54,103,142,162,177,179],"pairs":[43],"in":[44,92,120],"previous":[45],"distillation":[47,94,128],"methods.":[48],"However,":[49],"we":[50,123],"find":[51],"that":[52,192],"models":[56,105],"networks":[59,151],"or":[60],"trained":[61],"from":[62,112],"initialization":[64],"could":[65],"have":[66],"distinct":[67],"feature":[68],"representations":[69],"among":[70],"channels.":[72],"(e.g.":[73],"high":[75],"activated":[76],"channel":[77],"categories).":[80],"We":[81],"name":[82],"this":[83,118,121],"incongruous":[84],"representation":[85],"channels":[87],"as":[88],"teacher-student":[89],"discrepancy":[91,99,173],"process.":[95],"Ignoring":[96],"problem":[100,174],"will":[106],"make":[107],"learning":[109],"more":[114,137],"difficult.":[115],"To":[116],"solve":[117],"problem,":[119],"paper,":[122],"propose":[124],"novel":[126],"student-dependent":[127],"method,":[129],"consistent":[131,138],"distillation,":[132],"which":[133],"makes":[134],"teacher's":[135],"provides":[143],"best":[145],"suitable":[146],"distillation.":[153],"Extensive":[154],"experiments":[155],"on":[156],"datasets":[158],"(CIFAR100,":[159],"ImageNet,":[160],"COCO)":[161],"tasks":[163],"(image":[164],"classification,":[165],"object":[166],"detection)":[167],"reveal":[168],"widely":[170],"existing":[171],"between":[175],"teachers":[176],"students":[178],"demonstrate":[180],"effectiveness":[182],"our":[184],"proposed":[185],"method.":[186],"Our":[187],"method":[188],"very":[190],"flexible":[191],"can":[193],"be":[194],"easily":[195],"combined":[196],"other":[198],"state-of-the-art":[199],"approaches.":[200]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2021-04-13T00:00:00"}