{"id":"https://openalex.org/W3170224286","doi":"https://doi.org/10.1109/cvpr52688.2022.01065","title":"Knowledge distillation: A good teacher is patient and consistent","display_name":"Knowledge distillation: A good teacher is patient and consistent","publication_year":2022,"publication_date":"2022-06-01","ids":{"openalex":"https://openalex.org/W3170224286","doi":"https://doi.org/10.1109/cvpr52688.2022.01065","mag":"3170224286"},"language":"en","primary_location":{"id":"doi:10.1109/cvpr52688.2022.01065","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cvpr52688.2022.01065","pdf_url":null,"source":{"id":"https://openalex.org/S4363607701","display_name":"2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055629014","display_name":"Lucas Beyer","orcid":"https://orcid.org/0000-0002-0460-0607"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I4210113520","display_name":"Brain (Germany)","ror":"https://ror.org/01gamcy45","country_code":"DE","type":"company","lineage":["https://openalex.org/I4210113520"]}],"countries":["DE","US"],"is_corresponding":true,"raw_author_name":"Lucas Beyer","raw_affiliation_strings":["Google Research, Brain Team"],"affiliations":[{"raw_affiliation_string":"Google Research, Brain Team","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I4210113520"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071668416","display_name":"Xiaohua Zhai","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I4210113520","display_name":"Brain (Germany)","ror":"https://ror.org/01gamcy45","country_code":"DE","type":"company","lineage":["https://openalex.org/I4210113520"]}],"countries":["DE","US"],"is_corresponding":false,"raw_author_name":"Xiaohua Zhai","raw_affiliation_strings":["Google Research, Brain Team"],"affiliations":[{"raw_affiliation_string":"Google Research, Brain Team","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I4210113520"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034545872","display_name":"Am\u00e9lie Royer","orcid":"https://orcid.org/0000-0002-8407-0705"},"institutions":[{"id":"https://openalex.org/I4210113520","display_name":"Brain (Germany)","ror":"https://ror.org/01gamcy45","country_code":"DE","type":"company","lineage":["https://openalex.org/I4210113520"]},{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["DE","US"],"is_corresponding":false,"raw_author_name":"Amelie Royer","raw_affiliation_strings":["Google Research, Brain Team"],"affiliations":[{"raw_affiliation_string":"Google Research, Brain Team","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I4210113520"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003799439","display_name":"Larisa Markeeva","orcid":"https://orcid.org/0000-0002-7453-0827"},"institutions":[{"id":"https://openalex.org/I4210113520","display_name":"Brain (Germany)","ror":"https://ror.org/01gamcy45","country_code":"DE","type":"company","lineage":["https://openalex.org/I4210113520"]},{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["DE","US"],"is_corresponding":false,"raw_author_name":"Larisa Markeeva","raw_affiliation_strings":["Google Research, Brain Team"],"affiliations":[{"raw_affiliation_string":"Google Research, Brain Team","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I4210113520"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104083306","display_name":"Rohan Anil","orcid":null},"institutions":[{"id":"https://openalex.org/I4210113520","display_name":"Brain (Germany)","ror":"https://ror.org/01gamcy45","country_code":"DE","type":"company","lineage":["https://openalex.org/I4210113520"]},{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["DE","US"],"is_corresponding":false,"raw_author_name":"Rohan Anil","raw_affiliation_strings":["Google Research, Brain Team"],"affiliations":[{"raw_affiliation_string":"Google Research, Brain Team","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I4210113520"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5000026575","display_name":"Alexander Kolesnikov","orcid":null},"institutions":[{"id":"https://openalex.org/I4210113520","display_name":"Brain (Germany)","ror":"https://ror.org/01gamcy45","country_code":"DE","type":"company","lineage":["https://openalex.org/I4210113520"]},{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["DE","US"],"is_corresponding":false,"raw_author_name":"Alexander Kolesnikov","raw_affiliation_strings":["Google Research, Brain Team"],"affiliations":[{"raw_affiliation_string":"Google Research, Brain Team","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I4210113520"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5055629014"],"corresponding_institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I4210113520"],"apc_list":null,"apc_paid":null,"fwci":12.0678,"has_fulltext":false,"cited_by_count":211,"citation_normalized_percentile":{"value":0.98885735,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"10915","last_page":"10924"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7953665852546692},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.748551607131958},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.6379379034042358},{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.6246464252471924},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5980066061019897},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5882808566093445},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5673343539237976},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.5490139722824097},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5452476143836975},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.4718727469444275},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.415108859539032},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.11829811334609985},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.09450846910476685},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08786216378211975}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7953665852546692},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.748551607131958},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.6379379034042358},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.6246464252471924},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5980066061019897},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5882808566093445},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5673343539237976},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.5490139722824097},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5452476143836975},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4718727469444275},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.415108859539032},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.11829811334609985},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.09450846910476685},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08786216378211975},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/cvpr52688.2022.01065","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cvpr52688.2022.01065","pdf_url":null,"source":{"id":"https://openalex.org/S4363607701","display_name":"2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":93,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1690739335","https://openalex.org/W1821462560","https://openalex.org/W1836465849","https://openalex.org/W1977295328","https://openalex.org/W2017814585","https://openalex.org/W2025713906","https://openalex.org/W2039875163","https://openalex.org/W2097117768","https://openalex.org/W2117539524","https://openalex.org/W2167215970","https://openalex.org/W2194775991","https://openalex.org/W2286365479","https://openalex.org/W2549139847","https://openalex.org/W2592691248","https://openalex.org/W2606722458","https://openalex.org/W2612445135","https://openalex.org/W2748818695","https://openalex.org/W2752782242","https://openalex.org/W2765407302","https://openalex.org/W2769312802","https://openalex.org/W2908510526","https://openalex.org/W2922509574","https://openalex.org/W2945176031","https://openalex.org/W2950248853","https://openalex.org/W2953070460","https://openalex.org/W2954650510","https://openalex.org/W2955425717","https://openalex.org/W2963048316","https://openalex.org/W2963122961","https://openalex.org/W2963263347","https://openalex.org/W2963399829","https://openalex.org/W2963420686","https://openalex.org/W2963723401","https://openalex.org/W2964118293","https://openalex.org/W2964121744","https://openalex.org/W2971315489","https://openalex.org/W2981342956","https://openalex.org/W2982083293","https://openalex.org/W2991391304","https://openalex.org/W3004127093","https://openalex.org/W3017746288","https://openalex.org/W3035003500","https://openalex.org/W3035060554","https://openalex.org/W3035160371","https://openalex.org/W3038041907","https://openalex.org/W3048030262","https://openalex.org/W3084937072","https://openalex.org/W3094502228","https://openalex.org/W3097217077","https://openalex.org/W3111156583","https://openalex.org/W3133629262","https://openalex.org/W3134389780","https://openalex.org/W3138516171","https://openalex.org/W3138994021","https://openalex.org/W3157506437","https://openalex.org/W3168547821","https://openalex.org/W3176659256","https://openalex.org/W4285051865","https://openalex.org/W4287828539","https://openalex.org/W4293929015","https://openalex.org/W4297775537","https://openalex.org/W6631190155","https://openalex.org/W6637551013","https://openalex.org/W6638523607","https://openalex.org/W6638667902","https://openalex.org/W6679667936","https://openalex.org/W6684563725","https://openalex.org/W6696004547","https://openalex.org/W6726497184","https://openalex.org/W6733814495","https://openalex.org/W6737664043","https://openalex.org/W6745136726","https://openalex.org/W6745722055","https://openalex.org/W6747321236","https://openalex.org/W6749954789","https://openalex.org/W6756427901","https://openalex.org/W6757817989","https://openalex.org/W6762718338","https://openalex.org/W6763067651","https://openalex.org/W6763310536","https://openalex.org/W6774302960","https://openalex.org/W6775298725","https://openalex.org/W6776320331","https://openalex.org/W6779101013","https://openalex.org/W6779326418","https://openalex.org/W6781575362","https://openalex.org/W6782269215","https://openalex.org/W6787144432","https://openalex.org/W6791581255","https://openalex.org/W6791793911","https://openalex.org/W6791987244","https://openalex.org/W6795140394"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W4224009465","https://openalex.org/W4286629047","https://openalex.org/W2329452785","https://openalex.org/W2356380379","https://openalex.org/W4306321456","https://openalex.org/W4285260836","https://openalex.org/W3046775127","https://openalex.org/W3170094116"],"abstract_inverted_index":{"There":[0],"is":[1,119],"a":[2,52,59,83,141,149,159],"growing":[3],"discrepancy":[4],"in":[5,20,71,132,156],"computer":[6],"vision":[7,153],"between":[8,35],"large-scale":[9],"models":[10,16,69,92],"that":[11,17,101],"achieve":[12],"state-of-the-art":[13,66,160],"performance":[14],"and":[15,30,61],"are":[18,103],"affordable":[19,70],"practical":[21],"applications.":[22],"In":[23,97],"this":[24,28],"paper":[25],"we":[26,45,99],"address":[27],"issue":[29],"significantly":[31],"bridge":[32],"the":[33,88,112,120,133],"gap":[34],"these":[36,124],"two":[37],"types":[38],"of":[39,90,114,123,152],"models.":[40],"Throughout":[41],"our":[42,138],"empirical":[43,143],"investigation":[44],"do":[46],"not":[47,129],"aim":[48],"to":[49,57],"necessarily":[50],"propose":[51],"new":[53],"method,":[54],"but":[55],"strive":[56],"identify":[58],"robust":[60],"effective":[62],"recipe":[63],"for":[64,86,163],"making":[65],"large":[67,91],"scale":[68],"practice.":[72],"We":[73,135],"demonstrate":[74,145],"that,":[75],"when":[76],"performed":[77],"correctly,":[78],"knowledge":[79],"distillation":[80],"can":[81],"be":[82],"powerful":[84],"tool":[85],"reducing":[87],"size":[89],"without":[93],"compromising":[94],"their":[95],"performance.":[96],"particular,":[98,157],"uncover":[100],"there":[102],"certain":[104],"implicit":[105],"design":[106,125],"choices,":[107,126],"which":[108,127,165],"may":[109],"drastically":[110],"affect":[111],"effectiveness":[113],"distillation.":[115],"Our":[116],"key":[117],"contribution":[118],"explicit":[121],"identification":[122],"were":[128],"previously":[130],"articulated":[131],"literature.":[134],"back":[136],"up":[137],"findings":[139],"by":[140],"comprehensive":[142],"study,":[144],"compelling":[146],"results":[147],"on":[148],"wide":[150],"range":[151],"datasets":[154],"and,":[155],"obtain":[158],"ResNet-50":[161],"model":[162],"ImageNet,":[164],"achieves":[166],"82.8%":[167],"top-1":[168],"accuracy.":[169]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":56},{"year":2024,"cited_by_count":74},{"year":2023,"cited_by_count":59},{"year":2022,"cited_by_count":13},{"year":2021,"cited_by_count":5}],"updated_date":"2026-03-27T14:29:43.386196","created_date":"2025-10-10T00:00:00"}