{"id":"https://openalex.org/W3036049548","doi":null,"title":"Self-Knowledge Distillation: A Simple Way for Better Generalization","display_name":"Self-Knowledge Distillation: A Simple Way for Better Generalization","publication_year":2020,"publication_date":"2020-06-22","ids":{"openalex":"https://openalex.org/W3036049548","mag":"3036049548"},"language":"en","primary_location":{"id":"mag:3036049548","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2006.12000","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://export.arxiv.org/pdf/2006.12000","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052340301","display_name":"Kyungyul Kim","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kyungyul Kim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027410027","display_name":"Byeongmoon Ji","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Byeongmoon Ji","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078385419","display_name":"Doyoung Yoon","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Doyoung Yoon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5091438057","display_name":"Sangheum Hwang","orcid":"https://orcid.org/0000-0003-2136-296X","observed_orcids":["https://orcid.org/0000-0003-2136-296X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sangheum Hwang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":39,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.5473999977111816,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.5473999977111816,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.4025999903678894,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.01860000006854534,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/self-distillation","display_name":"self-distillation","score":0.9980000257492065},{"id":"https://openalex.org/keywords/knowledge-distillation","display_name":"knowledge distillation","score":0.9210000038146973},{"id":"https://openalex.org/keywords/generalization","display_name":"generalization","score":0.6470000147819519},{"id":"https://openalex.org/keywords/regularization","display_name":"regularization","score":0.7720000147819519},{"id":"https://openalex.org/keywords/soft-targets","display_name":"soft targets","score":0.7170000076293945},{"id":"https://openalex.org/keywords/teacher-student-models","display_name":"teacher-student models","score":0.5989999771118164},{"id":"https://openalex.org/keywords/confidence-estimation","display_name":"confidence estimation","score":0.9350000023841858},{"id":"https://openalex.org/keywords/image-classification","display_name":"image classification","score":0.871999979019165},{"id":"https://openalex.org/keywords/object-detection","display_name":"object detection","score":1.0},{"id":"https://openalex.org/keywords/machine-translation","display_name":"machine translation","score":1.0}],"concepts":[{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.7598621845245361},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7287275791168213},{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.6773754358291626},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5765498876571655},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.5550572276115417},{"id":"https://openalex.org/C125112378","wikidata":"https://www.wikidata.org/wiki/Q176640","display_name":"Randomness","level":2,"score":0.5426602363586426},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5275240540504456},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.19927909970283508},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0961957573890686},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.07473298907279968},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:3036049548","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2006.12000","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:3036049548","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2006.12000","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":36,"referenced_works":["https://openalex.org/W639708223","https://openalex.org/W1686810756","https://openalex.org/W1821462560","https://openalex.org/W2031489346","https://openalex.org/W2108598243","https://openalex.org/W2144513243","https://openalex.org/W2156297475","https://openalex.org/W2194775991","https://openalex.org/W2254249950","https://openalex.org/W2259472270","https://openalex.org/W2302255633","https://openalex.org/W2345720230","https://openalex.org/W2509282593","https://openalex.org/W2521284735","https://openalex.org/W2549139847","https://openalex.org/W2626778328","https://openalex.org/W2626967530","https://openalex.org/W2736941579","https://openalex.org/W2746314669","https://openalex.org/W2765407302","https://openalex.org/W2806070179","https://openalex.org/W2903105043","https://openalex.org/W2905927205","https://openalex.org/W2908336025","https://openalex.org/W2946948417","https://openalex.org/W2949117887","https://openalex.org/W2951093852","https://openalex.org/W2962971773","https://openalex.org/W2963446712","https://openalex.org/W2964121744","https://openalex.org/W2964268168","https://openalex.org/W2964508650","https://openalex.org/W2970206392","https://openalex.org/W2987861506","https://openalex.org/W3004809259","https://openalex.org/W3034695001"],"related_works":["https://openalex.org/W3204610735","https://openalex.org/W3200958619","https://openalex.org/W4372311908","https://openalex.org/W4212996245","https://openalex.org/W2904170036","https://openalex.org/W4313049560","https://openalex.org/W3206776650","https://openalex.org/W3034695001","https://openalex.org/W4376602333","https://openalex.org/W4403210108"],"abstract_inverted_index":{"The":[0,79],"generalization":[1,105],"capability":[2],"of":[3,16,69,122,147,158],"deep":[4],"neural":[5],"networks":[6],"has":[7],"been":[8],"substantially":[9],"improved":[10],"by":[11],"applying":[12],"a":[13,35,48,67,73,76],"wide":[14],"spectrum":[15],"regularization":[17,39,99],"methods,":[18],"e.g.,":[19],"restricting":[20],"function":[21],"space,":[22],"injecting":[23],"randomness":[24],"during":[25,59],"training,":[26],"augmenting":[27],"data,":[28],"etc.":[29],"In":[30],"this":[31],"work,":[32],"we":[33,108],"propose":[34],"simple":[36],"yet":[37],"effective":[38],"method":[40,81,142],"named":[41],"self-knowledge":[42],"distillation":[43,71],"(Self-KD),":[44],"which":[45],"progressively":[46],"distills":[47],"model's":[49],"own":[50],"knowledge":[51,70],"to":[52,84,101],"soften":[53],"hard":[54,90],"targets":[55,91],"(i.e.,":[56],"one-hot":[57],"vectors)":[58],"training.":[60],"Hence,":[61],"it":[62,153],"can":[63,93],"be":[64,94],"interpreted":[65],"within":[66],"framework":[68],"as":[72],"student":[74],"becomes":[75],"teacher":[77],"itself.":[78],"proposed":[80],"is":[82],"applicable":[83],"any":[85],"supervised":[86],"learning":[87],"tasks":[88],"with":[89,97],"and":[92,136,151,160,165],"easily":[95],"combined":[96],"existing":[98],"methods":[100],"further":[102],"enhance":[103],"the":[104,145,148],"performance.":[106],"Furthermore,":[107],"show":[109],"that":[110,140],"Self-KD":[111],"achieves":[112,154],"not":[113],"only":[114],"better":[115],"accuracy,":[116],"but":[117],"also":[118],"provides":[119],"high":[120],"quality":[121],"confidence":[123],"estimates.":[124],"Extensive":[125],"experimental":[126],"results":[127],"on":[128,162],"three":[129],"different":[130],"tasks,":[131,167],"image":[132],"classification,":[133],"object":[134],"detection,":[135],"machine":[137],"translation,":[138],"demonstrate":[139],"our":[141],"consistently":[143],"improves":[144],"performance":[146],"state-of-the-art":[149,155],"baselines,":[150],"especially,":[152],"BLEU":[156],"score":[157],"30.0":[159],"36.2":[161],"IWSLT15":[163],"English-to-German":[164],"German-to-English":[166],"respectively.":[168]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":18},{"year":2020,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2020-06-25T00:00:00"}