{"id":"https://openalex.org/W2945289329","doi":"https://doi.org/10.48550/arxiv.2105.13093","title":"Towards Understanding Knowledge Distillation","display_name":"Towards Understanding Knowledge Distillation","alternate_titles":[],"publication_year":2021,"publication_date":"2021-05-27","ids":{"openalex":"https://openalex.org/W2945289329","doi":"https://doi.org/10.48550/arxiv.2105.13093","mag":"2945289329"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2105.13093","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2105.13093","pdf_url":"https://arxiv.org/pdf/2105.13093","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W2945289329"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2105.13093","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034739012","display_name":"Mary Phuong","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157556583","display_name":"Institute of Science and Technology Austria","ror":"https://ror.org/03gnh5541","country_code":"AT","type":"education","lineage":["https://openalex.org/I157556583"]}],"countries":["AT"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Phuong, Mary","raw_affiliation_strings":["IST Austria ( Institute of Science and Technology Austria) ","Institute of Science and Technology Austria, Klosterneuburg, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IST Austria ( Institute of Science and Technology Austria) ","institution_ids":["https://openalex.org/I157556583"]},{"raw_affiliation_string":"Institute of Science and Technology Austria, Klosterneuburg, Austria","institution_ids":["https://openalex.org/I157556583"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068085751","display_name":"Christoph H. Lampert","orcid":"https://orcid.org/0000-0001-8622-7887","observed_orcids":["https://orcid.org/0000-0001-8622-7887"]},"institutions":[{"id":"https://openalex.org/I157556583","display_name":"Institute of Science and Technology Austria","ror":"https://ror.org/03gnh5541","country_code":"AT","type":"education","lineage":["https://openalex.org/I157556583"]}],"countries":["AT"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Lampert, Christoph H.","raw_affiliation_strings":["IST Austria ( Institute of Science and Technology Austria) ","Institute of Science and Technology Austria, Klosterneuburg, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IST Austria ( Institute of Science and Technology Austria) ","institution_ids":["https://openalex.org/I157556583"]},{"raw_affiliation_string":"Institute of Science and Technology Austria, Klosterneuburg, Austria","institution_ids":["https://openalex.org/I157556583"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I157556583"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":297,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"97","issue":null,"first_page":"5142","last_page":"5151"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.6547999978065491,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.6547999978065491,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.14000000059604645,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.09480000287294388,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/knowledge-distillation","display_name":"knowledge distillation","score":1.0},{"id":"https://openalex.org/keywords/soft-labels","display_name":"soft labels","score":0.7820000052452087},{"id":"https://openalex.org/keywords/generalization-bounds","display_name":"generalization bounds","score":0.8740000128746033},{"id":"https://openalex.org/keywords/linear-classifiers","display_name":"linear classifiers","score":0.8309999704360962},{"id":"https://openalex.org/keywords/data-geometry","display_name":"data geometry","score":0.8029999732971191},{"id":"https://openalex.org/keywords/class-separation","display_name":"class separation","score":0.9390000104904175},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"gradient descent","score":0.800000011920929}],"concepts":[{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.8261547684669495},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.7562447190284729},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6604633331298828},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6496405005455017},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5558183193206787},{"id":"https://openalex.org/C72169020","wikidata":"https://www.wikidata.org/wiki/Q194404","display_name":"Monotonic function","level":2,"score":0.509313702583313},{"id":"https://openalex.org/C117765406","wikidata":"https://www.wikidata.org/wiki/Q5362437","display_name":"Generalization error","level":3,"score":0.44261297583580017},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.42784932255744934},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.415999174118042},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.38349857926368713},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2938730716705322},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.07967045903205872},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"pmh:oai:arXiv.org:2105.13093","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2105.13093","pdf_url":"https://arxiv.org/pdf/2105.13093","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3165285531","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2105.13093","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2105.13093","doi":"https://doi.org/10.48550/arxiv.2105.13093","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2105.13093","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"mag:2945289329","doi":null,"is_oa":false,"landing_page_url":"http://proceedings.mlr.press/v97/phuong19a/phuong19a.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306419644","display_name":"International Conference on Machine Learning","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"International Conference on Machine Learning","raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2105.13093","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2105.13093","pdf_url":"https://arxiv.org/pdf/2105.13093","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2945289329.pdf","grobid_xml":"https://content.openalex.org/works/W2945289329.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W1602122992","https://openalex.org/W1690739335","https://openalex.org/W1821462560","https://openalex.org/W2048500237","https://openalex.org/W2113882472","https://openalex.org/W2134797427","https://openalex.org/W2173379916","https://openalex.org/W2174868984","https://openalex.org/W2182055801","https://openalex.org/W2232010405","https://openalex.org/W2253986341","https://openalex.org/W2293637555","https://openalex.org/W2294370754","https://openalex.org/W2404068508","https://openalex.org/W2558221787","https://openalex.org/W2565538933","https://openalex.org/W2584377191","https://openalex.org/W2592335154","https://openalex.org/W2612445135","https://openalex.org/W2787752464","https://openalex.org/W2962726660","https://openalex.org/W2962737704","https://openalex.org/W2963376662","https://openalex.org/W2963504252","https://openalex.org/W2963687836","https://openalex.org/W3018315329","https://openalex.org/W3119651796","https://openalex.org/W3145281271"],"related_works":["https://openalex.org/W3013255770","https://openalex.org/W7161126665","https://openalex.org/W4401864276","https://openalex.org/W3004809259","https://openalex.org/W2995105044","https://openalex.org/W4318751672","https://openalex.org/W4415002208","https://openalex.org/W7133221871","https://openalex.org/W3094460838"],"abstract_inverted_index":{"Knowledge":[0],"distillation,":[1],"i.e.,":[2],"one":[3],"classifier":[4,44,179],"being":[5],"trained":[6,38],"on":[7,145],"the":[8,40,71,75,82,102,111,123,134,146,150,165,173,177,183,186],"outputs":[9,41],"of":[10,42,49,63,78,85,101,105,125,133,149,164,176,185],"another":[11,43],"classifier,":[12],"is":[13,58],"an":[14],"empirically":[15],"very":[16,161],"successful":[17],"technique":[18],"for":[19],"knowledge":[20],"transfer":[21],"between":[22],"classifiers.":[23,90],"It":[24],"has":[25,141],"even":[26],"been":[27],"observed":[28],"that":[29,97,121],"classifiers":[30],"learn":[31],"much":[32],"faster":[33],"and":[34,87,113,168],"more":[35],"reliably":[36],"if":[37],"with":[39],"as":[45],"soft":[46],"labels,":[47],"instead":[48],"from":[50],"ground":[51],"truth":[52],"data.":[53],"So":[54],"far,":[55],"however,":[56],"there":[57],"no":[59],"satisfactory":[60],"theoretical":[61],"explanation":[62],"this":[64,67],"phenomenon.":[65],"In":[66],"work,":[68],"we":[69,92,116],"provide":[70],"first":[72],"insights":[73],"into":[74],"working":[76],"mechanisms":[77],"distillation":[79,166],"by":[80],"studying":[81],"special":[83],"case":[84],"linear":[86,89,108],"deep":[88],"Specifically,":[91],"prove":[93],"a":[94,106,142,160],"generalization":[95],"bound":[96,112],"establishes":[98],"fast":[99],"convergence":[100,147],"expected":[103,174],"risk":[104,175],"distillation-trained":[107],"classifier.":[109],"From":[110],"its":[114],"proof":[115],"extract":[117],"three":[118],"key":[119],"factors":[120],"determine":[122],"success":[124],"distillation:":[126],"*":[127,152,169],"data":[128,135],"geometry":[129],"--":[130,155,172],"geometric":[131],"properties":[132],"distribution,":[136],"in":[137],"particular":[138],"class":[139],"separation,":[140],"direct":[143],"influence":[144],"speed":[148],"risk;":[151],"optimization":[153,158],"bias":[154],"gradient":[156],"descent":[157],"finds":[159],"favorable":[162],"minimum":[163],"objective;":[167],"strong":[170],"monotonicity":[171],"student":[178],"always":[180],"decreases":[181],"when":[182],"size":[184],"training":[187],"set":[188],"grows.":[189]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2945289329","counts_by_year":[{"year":2026,"cited_by_count":39},{"year":2025,"cited_by_count":51},{"year":2024,"cited_by_count":38},{"year":2023,"cited_by_count":44},{"year":2022,"cited_by_count":40},{"year":2021,"cited_by_count":45},{"year":2020,"cited_by_count":40}],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2019-05-29T00:00:00"}
