{"id":"https://openalex.org/W2983118306","doi":null,"title":"Stagewise Knowledge Distillation","display_name":"Stagewise Knowledge Distillation","publication_year":2019,"publication_date":"2019-11-15","ids":{"openalex":"https://openalex.org/W2983118306","mag":"2983118306"},"language":"en","primary_location":{"id":"mag:2983118306","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/1911.06786v1","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/abs/1911.06786v1","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054750708","display_name":"Akshay Kulkarni","orcid":"https://orcid.org/0000-0003-3379-2238","observed_orcids":["https://orcid.org/0000-0003-3379-2238"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Akshay Kulkarni","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-3379-2238","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029830655","display_name":"Navid Panchi","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Navid Panchi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5045001583","display_name":"Shital S. Chiddarwar","orcid":"https://orcid.org/0000-0001-8078-6573","observed_orcids":["https://orcid.org/0000-0001-8078-6573"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shital S. Chiddarwar","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-8078-6573","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.009499999694526196,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.0012000000569969416,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/knowledge-distillation","display_name":"knowledge distillation","score":0.8479999899864197},{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.7179999947547913},{"id":"https://openalex.org/keywords/teacher-student-models","display_name":"teacher-student models","score":0.7360000014305115},{"id":"https://openalex.org/keywords/resource-constrained-systems","display_name":"resource-constrained systems","score":0.7929999828338623},{"id":"https://openalex.org/keywords/semantic-segmentation","display_name":"semantic segmentation","score":0.9079999923706055},{"id":"https://openalex.org/keywords/image-classification","display_name":"image classification","score":0.8640000224113464},{"id":"https://openalex.org/keywords/progressive-training","display_name":"progressive training","score":0.44200000166893005}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7571291327476501},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.7243450284004211},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6707668304443359},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5777484774589539},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.534242570400238},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.48836061358451843},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.470251202583313},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.4427139163017273},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4305564761161804},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.4284634292125702},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.18101227283477783},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08435681462287903},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C6557445","wikidata":"https://www.wikidata.org/wiki/Q173113","display_name":"Agronomy","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:2983118306","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/1911.06786v1","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:2983118306","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/1911.06786v1","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W1690739335","https://openalex.org/W1821462560","https://openalex.org/W2097117768","https://openalex.org/W2108598243","https://openalex.org/W2112796928","https://openalex.org/W2134797427","https://openalex.org/W2194775991","https://openalex.org/W2294370754","https://openalex.org/W2612864759","https://openalex.org/W2739879705","https://openalex.org/W2743289088","https://openalex.org/W2756085244","https://openalex.org/W2766839578","https://openalex.org/W2769312802","https://openalex.org/W2787752464","https://openalex.org/W2907257898","https://openalex.org/W2949117887","https://openalex.org/W2963397674","https://openalex.org/W2963446712","https://openalex.org/W2964121744","https://openalex.org/W2996569511","https://openalex.org/W3118608800"],"related_works":["https://openalex.org/W3036961584","https://openalex.org/W3007079638","https://openalex.org/W4285410134","https://openalex.org/W3038907258","https://openalex.org/W7133363105","https://openalex.org/W4224075564","https://openalex.org/W3091075517","https://openalex.org/W2972284200","https://openalex.org/W3131143179","https://openalex.org/W4400076706"],"abstract_inverted_index":{"Despite":[0],"the":[1,7,54,60,63,90,94,114,125,133],"success":[2],"of":[3,9,124],"Deep":[4],"Learning":[5],"(DL),":[6],"deployment":[8],"modern":[10],"DL":[11],"models":[12],"requiring":[13],"large":[14],"computational":[15],"power":[16],"poses":[17],"a":[18,49,70,122,157],"significant":[19,117],"problem":[20],"for":[21],"resource-constrained":[22],"systems.":[23],"This":[24],"necessitates":[25],"building":[26],"compact":[27],"networks":[28],"that":[29,40,146,162],"reduce":[30],"computations":[31],"while":[32,58],"preserving":[33],"performance.":[34],"Traditional":[35],"Knowledge":[36,75],"Distillation":[37,76],"(KD)":[38],"methods":[39,83,167],"transfer":[41],"knowledge":[42,61,91],"from":[43,93],"teacher":[44],"to":[45,62,88],"student":[46],"(a)":[47],"use":[48],"single-stage":[50],"and":[51,107,144],"(b)":[52],"require":[53],"whole":[55],"data":[56,126],"set":[57],"distilling":[59],"student.":[64],"In":[65],"this":[66],"work,":[67],"we":[68],"propose":[69],"new":[71],"method":[72,104,139,152],"called":[73],"Stagewise":[74],"(SKD)":[77],"which":[78],"builds":[79],"on":[80,105,132],"traditional":[81],"KD":[82,142],"by":[84],"progressive":[85],"stagewise":[86],"training":[87],"leverage":[89],"gained":[92],"teacher,":[95],"resulting":[96],"in":[97,128],"data-efficient":[98],"distillation":[99],"process.":[100],"We":[101,111,135],"evaluate":[102],"our":[103,138,151],"classification":[106],"semantic":[108],"segmentation":[109],"tasks.":[110],"show,":[112],"across":[113],"tested":[115],"tasks,":[116],"performance":[118],"gains":[119],"even":[120],"with":[121,140],"fraction":[123],"used":[127],"distillation,":[129],"without":[130],"compromising":[131],"metric.":[134],"also":[136],"compare":[137],"existing":[141],"techniques":[143],"show":[145],"SKD":[147],"outperforms":[148],"them.":[149],"Moreover,":[150],"can":[153],"be":[154],"viewed":[155],"as":[156,169],"generalized":[158],"model":[159,165],"compression":[160,166],"technique":[161],"complements":[163],"other":[164],"such":[168],"quantization":[170],"or":[171],"pruning.":[172]},"counts_by_year":[{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2019-11-22T00:00:00"}