{"id":"https://openalex.org/W4386895277","doi":"https://doi.org/10.3390/bdcc7030154","title":"Efficient and Controllable Model Compression through Sequential Knowledge Distillation and Pruning","display_name":"Efficient and Controllable Model Compression through Sequential Knowledge Distillation and Pruning","alternate_titles":[],"publication_year":2023,"publication_date":"2023-09-19","ids":{"openalex":"https://openalex.org/W4386895277","doi":"https://doi.org/10.3390/bdcc7030154"},"language":"en","primary_location":{"id":"doi:10.3390/bdcc7030154","doi":"https://doi.org/10.3390/bdcc7030154","is_oa":true,"landing_page_url":"https://doi.org/10.3390/bdcc7030154","pdf_url":"https://www.mdpi.com/2504-2289/7/3/154/pdf?version=1695126989","source":{"id":"https://openalex.org/S4210238752","display_name":"Big Data and Cognitive Computing","issn_l":"2504-2289","issn":["2504-2289"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","jufo-1","norway-1","poland-20","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Big Data and Cognitive Computing","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W4386895277"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["crossref","datacite","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2504-2289/7/3/154/pdf?version=1695126989","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5078387658","display_name":"Leila Malihi","orcid":"https://orcid.org/0009-0004-9362-8733","observed_orcids":["https://orcid.org/0009-0004-9362-8733"]},"institutions":[{"id":"https://openalex.org/I170658231","display_name":"Osnabr\u00fcck University","ror":"https://ror.org/04qmmjx98","country_code":"DE","type":"education","lineage":["https://openalex.org/I170658231"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Leila Malihi","raw_affiliation_strings":["Institute of Cognitive Science, Osnabr\u00fcck University, 49074 Osnabr\u00fcck, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Cognitive Science, Osnabr\u00fcck University, 49074 Osnabr\u00fcck, Germany","institution_ids":["https://openalex.org/I170658231"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036545001","display_name":"Gunther Heidemann","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I170658231","display_name":"Osnabr\u00fcck University","ror":"https://ror.org/04qmmjx98","country_code":"DE","type":"education","lineage":["https://openalex.org/I170658231"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Gunther Heidemann","raw_affiliation_strings":["Institute of Cognitive Science, Osnabr\u00fcck University, 49074 Osnabr\u00fcck, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Cognitive Science, Osnabr\u00fcck University, 49074 Osnabr\u00fcck, Germany","institution_ids":["https://openalex.org/I170658231"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5078387658"],"corresponding_institution_ids":["https://openalex.org/I170658231"],"apc_list":{"value":1600,"currency":"CHF","value_usd":1779},"apc_paid":{"value":1795,"currency":"EUR","value_usd":1935},"fwci":1.7729,"has_fulltext":true,"cited_by_count":28,"citation_normalized_percentile":{"value":0.86013411,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"7","issue":"3","first_page":"154","last_page":"154"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.9620000123977661},{"id":"https://openalex.org/keywords/knowledge-distillation","display_name":"knowledge distillation","score":0.9890000224113464},{"id":"https://openalex.org/keywords/network-pruning","display_name":"network pruning","score":0.9480000138282776},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"fine-tuning","score":0.9070000052452087},{"id":"https://openalex.org/keywords/resnet","display_name":"ResNet","score":0.6399999856948853},{"id":"https://openalex.org/keywords/efficientnet","display_name":"EfficientNet","score":0.6499999761581421},{"id":"https://openalex.org/keywords/densenet","display_name":"DenseNet","score":0.902999997138977},{"id":"https://openalex.org/keywords/cifar-10","display_name":"CIFAR-10","score":0.9589999914169312},{"id":"https://openalex.org/keywords/cifar-100","display_name":"CIFAR-100","score":0.996999979019165},{"id":"https://openalex.org/keywords/parameter-reduction","display_name":"parameter reduction","score":0.5540000200271606},{"id":"https://openalex.org/keywords/generalization","display_name":"generalization","score":0.6269999742507935}],"concepts":[{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.8534059524536133},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.7976257801055908},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7606019973754883},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.6928409337997437},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.5822737216949463},{"id":"https://openalex.org/C25797200","wikidata":"https://www.wikidata.org/wiki/Q828137","display_name":"Compression ratio","level":3,"score":0.50360506772995},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4670765697956085},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4661717712879181},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43906867504119873},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1665465533733368},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0968894362449646},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C511840579","wikidata":"https://www.wikidata.org/wiki/Q12757","display_name":"Internal combustion engine","level":2,"score":0.0},{"id":"https://openalex.org/C171146098","wikidata":"https://www.wikidata.org/wiki/Q124192","display_name":"Automotive engineering","level":1,"score":0.0},{"id":"https://openalex.org/C6557445","wikidata":"https://www.wikidata.org/wiki/Q173113","display_name":"Agronomy","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.3390/bdcc7030154","doi":"https://doi.org/10.3390/bdcc7030154","is_oa":true,"landing_page_url":"https://doi.org/10.3390/bdcc7030154","pdf_url":"https://www.mdpi.com/2504-2289/7/3/154/pdf?version=1695126989","source":{"id":"https://openalex.org/S4210238752","display_name":"Big Data and Cognitive Computing","issn_l":"2504-2289","issn":["2504-2289"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","jufo-1","norway-1","poland-20","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Big Data and Cognitive Computing","raw_type":"journal-article"},{"id":"pmh:oai:osnadocs.ub.uni-osnabrueck.de:ds-2024022810921","doi":null,"is_oa":true,"landing_page_url":"https://doi.org/10.48693/517","pdf_url":"https://osnadocs.ub.uni-osnabrueck.de/bitstream/ds-2024022810921/1/Malihi_Heidemann_BDCC-07-00154_2023.pdf","source":{"id":"https://openalex.org/S4377196933","display_name":"osnaDocs (Osnabr\u00fcck University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I170658231","host_organization_name":"Osnabr\u00fcck University","host_organization_lineage":["https://openalex.org/I170658231"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"doc-type:Article"},{"id":"pmh:oai:doaj.org/article:6a73d61954b74eac908d6f816b4ddb5d","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/6a73d61954b74eac908d6f816b4ddb5d","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Big Data and Cognitive Computing, Vol 7, Iss 3, p 154 (2023)","raw_type":"article"},{"id":"doi:10.48693/517","doi":"https://doi.org/10.48693/517","is_oa":true,"landing_page_url":"https://doi.org/10.48693/517","pdf_url":null,"source":{"id":"https://openalex.org/S4377196933","display_name":"osnaDocs (Osnabr\u00fcck University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I170658231","host_organization_name":"Osnabr\u00fcck University","host_organization_lineage":["https://openalex.org/I170658231"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Other"}],"best_oa_location":{"id":"doi:10.3390/bdcc7030154","doi":"https://doi.org/10.3390/bdcc7030154","is_oa":true,"landing_page_url":"https://doi.org/10.3390/bdcc7030154","pdf_url":"https://www.mdpi.com/2504-2289/7/3/154/pdf?version=1695126989","source":{"id":"https://openalex.org/S4210238752","display_name":"Big Data and Cognitive Computing","issn_l":"2504-2289","issn":["2504-2289"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","jufo-1","norway-1","poland-20","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Big Data and Cognitive Computing","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320329560","display_name":"Universit\u00e4t Osnabr\u00fcck","ror":"https://ror.org/04qmmjx98"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386895277.pdf","grobid_xml":"https://content.openalex.org/works/W4386895277.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W1821462560","https://openalex.org/W2119144962","https://openalex.org/W2515385951","https://openalex.org/W2561238782","https://openalex.org/W2748787960","https://openalex.org/W2786054724","https://openalex.org/W2805003733","https://openalex.org/W2936864631","https://openalex.org/W2944355599","https://openalex.org/W2955419283","https://openalex.org/W2964161024","https://openalex.org/W2981794819","https://openalex.org/W3042620538","https://openalex.org/W3119460773","https://openalex.org/W3171155106","https://openalex.org/W3174102142","https://openalex.org/W3174517946","https://openalex.org/W3176213365","https://openalex.org/W4312929415","https://openalex.org/W4362676543","https://openalex.org/W4387789922"],"related_works":["https://openalex.org/W4400246032","https://openalex.org/W4408219989","https://openalex.org/W7116096181","https://openalex.org/W3105082838","https://openalex.org/W4406281539","https://openalex.org/W3001327261","https://openalex.org/W7204448788","https://openalex.org/W4283328744","https://openalex.org/W4402091721","https://openalex.org/W2949393176"],"abstract_inverted_index":{"Efficient":[0],"model":[1,49,60,93],"deployment":[2],"is":[3,83],"a":[4],"key":[5],"focus":[6],"in":[7],"deep":[8],"learning.":[9],"This":[10,108],"has":[11],"led":[12],"to":[13,25,46,69,111,165],"the":[14,37,77,103,154,166],"exploration":[15],"of":[16,79,105,117,138],"methods":[17],"such":[18],"as":[19,123],"knowledge":[20,41,64],"distillation":[21,42],"and":[22,28,43,51,67,89,98,143,156],"network":[23,44],"pruning":[24,45],"compress":[26],"models":[27,113,159,169],"increase":[29],"their":[30],"performance.":[31],"In":[32,151],"this":[33],"study,":[34],"we":[35,129],"investigate":[36],"potential":[38],"synergy":[39],"between":[40],"achieve":[47,70],"optimal":[48],"efficiency":[50],"improved":[52],"generalization.":[53],"We":[54,100],"introduce":[55],"an":[56],"innovative":[57],"framework":[58],"for":[59],"compression":[61,72,106,118],"that":[62],"combines":[63],"distillation,":[65],"pruning,":[66],"fine-tuning":[68],"enhanced":[71],"while":[73,119,170],"providing":[74],"control":[75],"over":[76],"degree":[78],"compactness.":[80],"Our":[81,146],"research":[82],"conducted":[84],"on":[85],"popular":[86],"datasets,":[87],"CIFAR-10":[88],"CIFAR-100,":[90],"employing":[91],"diverse":[92],"architectures,":[94],"including":[95],"ResNet,":[96],"DenseNet,":[97],"EfficientNet.":[99],"could":[101],"calibrate":[102],"amount":[104],"achieved.":[107],"allows":[109],"us":[110],"produce":[112],"with":[114],"different":[115],"degrees":[116],"still":[120],"being":[121],"just":[122],"accurate,":[124],"or":[125,162],"even":[126],"better.":[127],"Notably,":[128],"demonstrate":[130],"its":[131],"efficacy":[132],"by":[133],"producing":[134],"two":[135],"compressed":[136],"variants":[137],"ResNet":[139,141,144],"101:":[140],"50":[142],"18.":[145],"results":[147],"reveal":[148],"intriguing":[149],"findings.":[150],"most":[152],"cases,":[153],"pruned":[155],"distilled":[157,167],"student":[158,168],"exhibit":[160],"comparable":[161],"superior":[163],"accuracy":[164],"utilizing":[171],"significantly":[172],"fewer":[173],"parameters.":[174]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4386895277","counts_by_year":[{"year":2026,"cited_by_count":12},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":4}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2023-09-21T00:00:00"}
