{"id":"https://openalex.org/W3042439022","doi":"https://doi.org/10.1109/access.2020.3008854","title":"Variational Bayesian Group-Level Sparsification for Knowledge Distillation","display_name":"Variational Bayesian Group-Level Sparsification for Knowledge Distillation","publication_year":2020,"publication_date":"2020-01-01","ids":{"openalex":"https://openalex.org/W3042439022","doi":"https://doi.org/10.1109/access.2020.3008854","mag":"3042439022"},"language":"en","primary_location":{"id":"doi:10.1109/access.2020.3008854","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2020.3008854","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09139512.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09139512.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089582342","display_name":"Yue Ming","orcid":"https://orcid.org/0000-0001-7105-4207"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yue Ming","raw_affiliation_strings":["Beijing Key Laboratory of Work Safety and Intelligent Monitoring, School of Electronic Engineering, Beijing University of Posts and Telecommunications, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Beijing Key Laboratory of Work Safety and Intelligent Monitoring, School of Electronic Engineering, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101611864","display_name":"Hao Fu","orcid":"https://orcid.org/0000-0003-3755-8756"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hao Fu","raw_affiliation_strings":["Beijing Key Laboratory of Work Safety and Intelligent Monitoring, School of Electronic Engineering, Beijing University of Posts and Telecommunications, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Beijing Key Laboratory of Work Safety and Intelligent Monitoring, School of Electronic Engineering, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101795804","display_name":"Yibo Jiang","orcid":"https://orcid.org/0000-0002-4276-7336"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yibo Jiang","raw_affiliation_strings":["China Ningbo XiTang Information Technologies, Inc., Ningbo, China"],"affiliations":[{"raw_affiliation_string":"China Ningbo XiTang Information Technologies, Inc., Ningbo, China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5006580423","display_name":"Hui Yu","orcid":"https://orcid.org/0000-0002-7655-9228"},"institutions":[{"id":"https://openalex.org/I63072094","display_name":"University of Portsmouth","ror":"https://ror.org/03ykbk197","country_code":"GB","type":"education","lineage":["https://openalex.org/I63072094"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Hui Yu","raw_affiliation_strings":["School of Creative Technologies, University of Portsmouth, Portsmouth, U.K"],"affiliations":[{"raw_affiliation_string":"School of Creative Technologies, University of Portsmouth, Portsmouth, U.K","institution_ids":["https://openalex.org/I63072094"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5089582342"],"corresponding_institution_ids":["https://openalex.org/I139759216"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.06775235,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"8","issue":null,"first_page":"126628","last_page":"126636"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mnist-database","display_name":"MNIST database","score":0.8839403390884399},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6973519325256348},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6341090798377991},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5584179759025574},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5541288256645203},{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.4841490387916565},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.4801639914512634},{"id":"https://openalex.org/keywords/bayesian-inference","display_name":"Bayesian inference","score":0.4683721959590912},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.45824357867240906},{"id":"https://openalex.org/keywords/boosting","display_name":"Boosting (machine learning)","score":0.44787412881851196},{"id":"https://openalex.org/keywords/bayesian-network","display_name":"Bayesian network","score":0.43896791338920593},{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.4389171898365021},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3549683094024658}],"concepts":[{"id":"https://openalex.org/C190502265","wikidata":"https://www.wikidata.org/wiki/Q17069496","display_name":"MNIST database","level":3,"score":0.8839403390884399},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6973519325256348},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6341090798377991},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5584179759025574},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5541288256645203},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.4841490387916565},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4801639914512634},{"id":"https://openalex.org/C160234255","wikidata":"https://www.wikidata.org/wiki/Q812535","display_name":"Bayesian inference","level":3,"score":0.4683721959590912},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.45824357867240906},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.44787412881851196},{"id":"https://openalex.org/C33724603","wikidata":"https://www.wikidata.org/wiki/Q812540","display_name":"Bayesian network","level":2,"score":0.43896791338920593},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.4389171898365021},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3549683094024658},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/access.2020.3008854","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2020.3008854","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09139512.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:d8d7ee4e60b144b7b9f9d6f5edefc2d8","is_oa":true,"landing_page_url":"https://doaj.org/article/d8d7ee4e60b144b7b9f9d6f5edefc2d8","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 8, Pp 126628-126636 (2020)","raw_type":"article"},{"id":"pmh:oai:researchportal.port.ac.uk:publications/2dded55d-806e-4a38-9787-7a341d4c162d","is_oa":true,"landing_page_url":"https://researchportal.port.ac.uk/portal/en/publications/variational-bayesian-grouplevel-sparsification-for-knowledge-distillation(2dded55d-806e-4a38-9787-7a341d4c162d).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306401774","display_name":"Portsmouth Research Portal (University of Portsmouth)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I63072094","host_organization_name":"University of Portsmouth","host_organization_lineage":["https://openalex.org/I63072094"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":""}],"best_oa_location":{"id":"doi:10.1109/access.2020.3008854","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2020.3008854","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09139512.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.5299999713897705}],"awards":[{"id":"https://openalex.org/G2087396116","display_name":null,"funder_award_id":"China","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2376276132","display_name":null,"funder_award_id":"China","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"},{"id":"https://openalex.org/G3085993365","display_name":null,"funder_award_id":"(Grant No.","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3317480652","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3710712069","display_name":null,"funder_award_id":"L182033","funder_id":"https://openalex.org/F4320322919","funder_display_name":"Natural Science Foundation of Beijing Municipality"},{"id":"https://openalex.org/G391238517","display_name":null,"funder_award_id":", and","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4515545457","display_name":null,"funder_award_id":"L182033","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"},{"id":"https://openalex.org/G5400356607","display_name":null,"funder_award_id":"L182033","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5994120800","display_name":null,"funder_award_id":"Natural","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7726157001","display_name":null,"funder_award_id":"Grant No.","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8095539393","display_name":null,"funder_award_id":"2019PTB-001","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8478882202","display_name":null,"funder_award_id":"(2019","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"},{"id":"https://openalex.org/G8873210102","display_name":null,"funder_award_id":"2019PTB-001","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"},{"id":"https://openalex.org/G8951484681","display_name":null,"funder_award_id":"Grant","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322919","display_name":"Natural Science Foundation of Beijing Municipality","ror":null},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3042439022.pdf","grobid_xml":"https://content.openalex.org/works/W3042439022.grobid-xml"},"referenced_works_count":56,"referenced_works":["https://openalex.org/W566555209","https://openalex.org/W1686810756","https://openalex.org/W1821462560","https://openalex.org/W2047229728","https://openalex.org/W2112984492","https://openalex.org/W2119144962","https://openalex.org/W2164129707","https://openalex.org/W2167215970","https://openalex.org/W2524428287","https://openalex.org/W2582745083","https://openalex.org/W2619890685","https://openalex.org/W2623451521","https://openalex.org/W2739879705","https://openalex.org/W2756085244","https://openalex.org/W2764043458","https://openalex.org/W2783873922","https://openalex.org/W2807912816","https://openalex.org/W2856871277","https://openalex.org/W2886756692","https://openalex.org/W2912476735","https://openalex.org/W2939308562","https://openalex.org/W2949964376","https://openalex.org/W2950248853","https://openalex.org/W2952088488","https://openalex.org/W2953725248","https://openalex.org/W2962965870","https://openalex.org/W2963145730","https://openalex.org/W2963387524","https://openalex.org/W2963516298","https://openalex.org/W2963656031","https://openalex.org/W2963674932","https://openalex.org/W2963991999","https://openalex.org/W2964203871","https://openalex.org/W2964220233","https://openalex.org/W2964299589","https://openalex.org/W2968868585","https://openalex.org/W2979303251","https://openalex.org/W3024182761","https://openalex.org/W3100877062","https://openalex.org/W3118608800","https://openalex.org/W6637373629","https://openalex.org/W6638523607","https://openalex.org/W6677580257","https://openalex.org/W6684563725","https://openalex.org/W6686067075","https://openalex.org/W6726275242","https://openalex.org/W6727208969","https://openalex.org/W6732814185","https://openalex.org/W6739000085","https://openalex.org/W6739513683","https://openalex.org/W6744495609","https://openalex.org/W6745148473","https://openalex.org/W6748163181","https://openalex.org/W6748171661","https://openalex.org/W6753087691","https://openalex.org/W6787972765"],"related_works":["https://openalex.org/W4386603768","https://openalex.org/W2950475743","https://openalex.org/W2886711096","https://openalex.org/W2750384547","https://openalex.org/W4380078352","https://openalex.org/W3046591097","https://openalex.org/W4389249638","https://openalex.org/W2733410219","https://openalex.org/W2734358244","https://openalex.org/W4388700941"],"abstract_inverted_index":{"Deep":[0],"neural":[1],"networks":[2,128],"are":[3],"capable":[4],"of":[5,25,86,108,171],"learning":[6],"powerful":[7],"representation,":[8],"but":[9],"often":[10],"limited":[11],"by":[12],"heavy":[13],"network":[14,67,74],"architectures":[15],"and":[16,33,71,92,134,150],"high":[17],"computational":[18],"cost.":[19],"Knowledge":[20,59],"distillation":[21],"(KD)":[22],"is":[23,113],"one":[24],"the":[26,37,80,84,89,94,125,139,148,165,169,172],"effective":[27],"ways":[28],"to":[29,62,98],"perform":[30],"model":[31],"compression":[32],"inference":[34],"acceleration.":[35],"But":[36],"final":[38],"student":[39,73,90,127],"models":[40],"remain":[41],"parameter":[42],"redundancy.":[43],"To":[44],"tackle":[45],"these":[46],"issues,":[47],"we":[48],"propose":[49],"a":[50,64,69],"novel":[51],"approach,":[52],"called":[53],"Variational":[54],"Bayesian":[55,96],"Group-level":[56],"Sparsification":[57],"for":[58],"Distillation":[60],"(VBGS-KD),":[61],"distill":[63],"large":[65],"teacher":[66],"into":[68],"small":[70],"sparse":[72],"while":[75],"preserving":[76],"accuracy.":[77],"We":[78,142],"impose":[79],"sparsity-inducing":[81],"prior":[82],"on":[83,147,164],"groups":[85],"parameters":[87],"in":[88,160],"model,":[91],"introduce":[93],"variational":[95],"approximation":[97],"learn":[99,124],"structural":[100],"sparseness,":[101],"which":[102],"can":[103,123],"effectively":[104],"prune":[105,111],"most":[106],"part":[107],"weights.":[109],"The":[110,120],"threshold":[112],"learned":[114],"during":[115],"training":[116],"without":[117],"extra":[118],"fine-tuning.":[119],"proposed":[121,173],"method":[122,146],"robust":[126],"that":[129],"have":[130,143],"achieved":[131],"satisifying":[132],"accuracy":[133,158],"compact":[135],"sizes":[136],"compared":[137],"with":[138,156],"state-of-the-arts":[140],"methods.":[141],"validated":[144],"our":[145],"MNIST":[149],"CIFAR-10":[151,166],"datasets,":[152],"observing":[153],"90.3%":[154],"sparsity":[155],"0.19%":[157],"boosting":[159],"MNIST.":[161],"Extensive":[162],"experiments":[163],"dataset":[167],"demonstrate":[168],"efficiency":[170],"approach.":[174]},"counts_by_year":[],"updated_date":"2026-03-18T14:38:29.013473","created_date":"2025-10-10T00:00:00"}