{"id":"https://openalex.org/W4224075564","doi":"https://doi.org/10.1109/tnnls.2022.3164264","title":"STKD: Distilling Knowledge From Synchronous Teaching for Efficient Model Compression","display_name":"STKD: Distilling Knowledge From Synchronous Teaching for Efficient Model Compression","publication_year":2022,"publication_date":"2022-04-14","ids":{"openalex":"https://openalex.org/W4224075564","doi":"https://doi.org/10.1109/tnnls.2022.3164264","pmid":"https://pubmed.ncbi.nlm.nih.gov/35420989"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2022.3164264","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2022.3164264","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Tongtong Su","orcid":"https://orcid.org/0000-0003-4021-6466"},"institutions":[{"id":"https://openalex.org/I205237279","display_name":"Nankai University","ror":"https://ror.org/01y1kjr75","country_code":"CN","type":"education","lineage":["https://openalex.org/I205237279"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Tongtong Su","raw_affiliation_strings":["TJ Key Laboratory of NDST, College of Computer Science, Nankai University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"TJ Key Laboratory of NDST, College of Computer Science, Nankai University, Tianjin, China","institution_ids":["https://openalex.org/I205237279"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Jinsong Zhang","orcid":"https://orcid.org/0000-0001-6238-7931"},"institutions":[{"id":"https://openalex.org/I205237279","display_name":"Nankai University","ror":"https://ror.org/01y1kjr75","country_code":"CN","type":"education","lineage":["https://openalex.org/I205237279"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jinsong Zhang","raw_affiliation_strings":["TJ Key Laboratory of NDST, College of Computer Science, Nankai University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"TJ Key Laboratory of NDST, College of Computer Science, Nankai University, Tianjin, China","institution_ids":["https://openalex.org/I205237279"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhaoyang Yu","orcid":"https://orcid.org/0000-0002-2450-4997"},"institutions":[{"id":"https://openalex.org/I205237279","display_name":"Nankai University","ror":"https://ror.org/01y1kjr75","country_code":"CN","type":"education","lineage":["https://openalex.org/I205237279"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhaoyang Yu","raw_affiliation_strings":["TJ Key Laboratory of NDST, College of Computer Science, Nankai University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"TJ Key Laboratory of NDST, College of Computer Science, Nankai University, Tianjin, China","institution_ids":["https://openalex.org/I205237279"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Gang Wang","orcid":"https://orcid.org/0000-0003-0387-2501"},"institutions":[{"id":"https://openalex.org/I205237279","display_name":"Nankai University","ror":"https://ror.org/01y1kjr75","country_code":"CN","type":"education","lineage":["https://openalex.org/I205237279"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Gang Wang","raw_affiliation_strings":["TJ Key Laboratory of NDST, College of Computer Science, Nankai University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"TJ Key Laboratory of NDST, College of Computer Science, Nankai University, Tianjin, China","institution_ids":["https://openalex.org/I205237279"]}]},{"author_position":"last","author":{"id":null,"display_name":"Xiaoguang Liu","orcid":"https://orcid.org/0000-0002-9010-3278"},"institutions":[{"id":"https://openalex.org/I205237279","display_name":"Nankai University","ror":"https://ror.org/01y1kjr75","country_code":"CN","type":"education","lineage":["https://openalex.org/I205237279"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoguang Liu","raw_affiliation_strings":["TJ Key Laboratory of NDST, College of Computer Science, Nankai University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"TJ Key Laboratory of NDST, College of Computer Science, Nankai University, Tianjin, China","institution_ids":["https://openalex.org/I205237279"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I205237279"],"apc_list":null,"apc_paid":null,"fwci":1.7334,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.85533265,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"34","issue":"12","first_page":"10051","last_page":"10064"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.3336000144481659,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.3336000144481659,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.18150000274181366,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.12470000237226486,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/discriminative-model","display_name":"Discriminative model","score":0.6690000295639038},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6421999931335449},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4993000030517578},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.477400004863739},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4278999865055084},{"id":"https://openalex.org/keywords/knowledge-extraction","display_name":"Knowledge extraction","score":0.38089999556541443},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.36059999465942383},{"id":"https://openalex.org/keywords/knowledge-acquisition","display_name":"Knowledge acquisition","score":0.3402999937534332},{"id":"https://openalex.org/keywords/table","display_name":"Table (database)","score":0.3278000056743622}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8414000272750854},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.6690000295639038},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6421999931335449},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5267999768257141},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5138000249862671},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4993000030517578},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.477400004863739},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4278999865055084},{"id":"https://openalex.org/C120567893","wikidata":"https://www.wikidata.org/wiki/Q1582085","display_name":"Knowledge extraction","level":2,"score":0.38089999556541443},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.36059999465942383},{"id":"https://openalex.org/C2777220311","wikidata":"https://www.wikidata.org/wiki/Q6423340","display_name":"Knowledge acquisition","level":2,"score":0.3402999937534332},{"id":"https://openalex.org/C45235069","wikidata":"https://www.wikidata.org/wiki/Q278425","display_name":"Table (database)","level":2,"score":0.3278000056743622},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.326200008392334},{"id":"https://openalex.org/C2780102126","wikidata":"https://www.wikidata.org/wiki/Q10928179","display_name":"Online and offline","level":2,"score":0.32170000672340393},{"id":"https://openalex.org/C2780490138","wikidata":"https://www.wikidata.org/wiki/Q7079636","display_name":"Offline learning","level":3,"score":0.30140000581741333},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.28929999470710754},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.28790000081062317},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.2680000066757202},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.2669000029563904},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.26460000872612},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.26170000433921814},{"id":"https://openalex.org/C56289545","wikidata":"https://www.wikidata.org/wiki/Q6423376","display_name":"Knowledge integration","level":3,"score":0.25999999046325684},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.2597000002861023},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2572000026702881},{"id":"https://openalex.org/C84685590","wikidata":"https://www.wikidata.org/wiki/Q1540472","display_name":"Knowledge engineering","level":2,"score":0.25690001249313354},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.2556000053882599}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2022.3164264","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2022.3164264","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:35420989","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/35420989","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":67,"referenced_works":["https://openalex.org/W1821462560","https://openalex.org/W1928906481","https://openalex.org/W2007339694","https://openalex.org/W2077319423","https://openalex.org/W2108598243","https://openalex.org/W2194775991","https://openalex.org/W2331143823","https://openalex.org/W2480078828","https://openalex.org/W2565639579","https://openalex.org/W2620998106","https://openalex.org/W2622826443","https://openalex.org/W2739879705","https://openalex.org/W2748787960","https://openalex.org/W2763160469","https://openalex.org/W2776460497","https://openalex.org/W2783231089","https://openalex.org/W2783538964","https://openalex.org/W2807912816","https://openalex.org/W2894994475","https://openalex.org/W2910121883","https://openalex.org/W2916798096","https://openalex.org/W2945289329","https://openalex.org/W2962851801","https://openalex.org/W2963140444","https://openalex.org/W2963446712","https://openalex.org/W2963503540","https://openalex.org/W2963521187","https://openalex.org/W2964137095","https://openalex.org/W2964221239","https://openalex.org/W2964233199","https://openalex.org/W2965502995","https://openalex.org/W2982242214","https://openalex.org/W2984618279","https://openalex.org/W2987861506","https://openalex.org/W2996970889","https://openalex.org/W3004127093","https://openalex.org/W3015735225","https://openalex.org/W3034528892","https://openalex.org/W3034674511","https://openalex.org/W3034795332","https://openalex.org/W3035321581","https://openalex.org/W3042011474","https://openalex.org/W3088199650","https://openalex.org/W3105082838","https://openalex.org/W3139445856","https://openalex.org/W3161758233","https://openalex.org/W4206913945","https://openalex.org/W6631190155","https://openalex.org/W6637373629","https://openalex.org/W6637551013","https://openalex.org/W6703116779","https://openalex.org/W6730179637","https://openalex.org/W6740493225","https://openalex.org/W6743188669","https://openalex.org/W6745722055","https://openalex.org/W6748082341","https://openalex.org/W6748108687","https://openalex.org/W6748163181","https://openalex.org/W6750401872","https://openalex.org/W6752523896","https://openalex.org/W6755034786","https://openalex.org/W6770540178","https://openalex.org/W6773204560","https://openalex.org/W6773604739","https://openalex.org/W6778160638","https://openalex.org/W6783655119","https://openalex.org/W6787972765"],"related_works":[],"abstract_inverted_index":{"Knowledge":[0],"distillation":[1,69],"(KD)":[2],"transfers":[3],"discriminative":[4],"knowledge":[5,38,51,68,83,101,171],"from":[6],"a":[7,16,63,92,146],"large":[8],"and":[9,18,75,81,102,113,127,154,169,176,198,222],"complex":[10],"model":[11,223],"(known":[12,21],"as":[13,22],"teacher)":[14],"to":[15,29,39,48,52,71,84,97,123,130,149],"smaller":[17],"faster":[19],"one":[20],"student).":[23],"Existing":[24],"advanced":[25],"KD":[26,185],"methods,":[27,217],"limited":[28],"fixed":[30],"feature":[31],"extraction":[32],"paradigms":[33],"that":[34,205],"capture":[35],"teacher's":[36],"structure":[37],"guide":[40],"the":[41,44,53,85,88,99,111,124,132,138,142,152,156,165,181,194,206,215],"training":[42,157],"of":[43,110,134,183],"student,":[45],"often":[46],"fail":[47],"obtain":[49],"comprehensive":[50,82,170],"student.":[54,86],"Toward":[55],"this":[56,59],"end,":[57],"in":[58],"article,":[60],"we":[61],"propose":[62],"new":[64],"approach,":[65],"synchronous":[66],"teaching":[67,74,77,175],"(STKD),":[70],"integrate":[72],"online":[73,89,174],"offline":[76,139,177],"for":[78,226],"transferring":[79],"rich":[80],"In":[87,137],"learning":[90,140],"stage,":[91,141],"blockwise":[93],"unit":[94],"is":[95,128,228],"designed":[96],"distill":[98],"intermediate-level":[100],"high-level":[103],"knowledge,":[104],"which":[105,179],"can":[106],"achieve":[107],"bidirectional":[108],"guidance":[109],"teacher":[112,148],"student":[114,125,166],"networks.":[115],"Intermediate-level":[116],"information":[117,122],"interaction":[118],"provides":[119],"more":[120],"supervisory":[121],"network":[126],"useful":[129],"enhance":[131],"quality":[133],"final":[135],"predictions.":[136],"STKD":[143,189,227],"approach":[144],"applies":[145],"pretrained":[147],"further":[150],"improve":[151],"performance":[153,211],"accelerate":[155],"process":[158],"by":[159,172],"providing":[160],"prior":[161],"knowledge.":[162],"Trained":[163],"simultaneously,":[164],"learns":[167],"multilevel":[168],"incorporating":[173],"teaching,":[178],"combines":[180],"advantages":[182],"different":[184],"strategies":[186],"through":[187],"our":[188],"method.":[190],"Experimental":[191],"results":[192],"on":[193],"SVHN,":[195],"CIFAR-10,":[196],"CIFAR-100,":[197],"ImageNet":[199],"ILSVRC":[200],"2012":[201],"real-world":[202],"datasets":[203],"show":[204],"proposed":[207],"method":[208],"achieves":[209],"significant":[210],"improvements":[212],"compared":[213],"with":[214,219],"state-of-the-art":[216],"especially":[218],"satisfying":[220],"accuracy":[221],"size.":[224],"Code":[225],"provided":[229],"at":[230],"https://github.com/nanxiaotong/STKD.":[231]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":1}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2022-04-19T00:00:00"}