{"id":"https://openalex.org/W3152578659","doi":"https://doi.org/10.3390/s21082792","title":"Memory-Replay Knowledge Distillation","display_name":"Memory-Replay Knowledge Distillation","publication_year":2021,"publication_date":"2021-04-15","ids":{"openalex":"https://openalex.org/W3152578659","doi":"https://doi.org/10.3390/s21082792","mag":"3152578659","pmid":"https://pubmed.ncbi.nlm.nih.gov/33921068"},"language":"en","primary_location":{"id":"doi:10.3390/s21082792","is_oa":true,"landing_page_url":"https://doi.org/10.3390/s21082792","pdf_url":"https://www.mdpi.com/1424-8220/21/8/2792/pdf?version=1619101797","source":{"id":"https://openalex.org/S101949793","display_name":"Sensors","issn_l":"1424-8220","issn":["1424-8220"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Sensors","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/1424-8220/21/8/2792/pdf?version=1619101797","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101963530","display_name":"Jiyue Wang","orcid":"https://orcid.org/0000-0002-2779-5355"},"institutions":[{"id":"https://openalex.org/I90610280","display_name":"South China University of Technology","ror":"https://ror.org/0530pts50","country_code":"CN","type":"education","lineage":["https://openalex.org/I90610280"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Jiyue Wang","raw_affiliation_strings":["School of Electronic and Information Engineering, South China University of Technology, Guangzhou 510641, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Electronic and Information Engineering, South China University of Technology, Guangzhou 510641, China","institution_ids":["https://openalex.org/I90610280"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031510879","display_name":"Pei Zhang","orcid":"https://orcid.org/0000-0002-6372-5653"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Pei Zhang","raw_affiliation_strings":["School of Computer Science, Northwestern Polytechnical University, Xi\u2019an 710072, China","School of Computer Science, Northwestern Polytechnical University, Xi'an 710072, China"],"raw_orcid":"https://orcid.org/0000-0002-6372-5653","affiliations":[{"raw_affiliation_string":"School of Computer Science, Northwestern Polytechnical University, Xi\u2019an 710072, China","institution_ids":["https://openalex.org/I17145004"]},{"raw_affiliation_string":"School of Computer Science, Northwestern Polytechnical University, Xi'an 710072, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070863631","display_name":"Yanxiong Li","orcid":"https://orcid.org/0000-0003-4362-1125"},"institutions":[{"id":"https://openalex.org/I90610280","display_name":"South China University of Technology","ror":"https://ror.org/0530pts50","country_code":"CN","type":"education","lineage":["https://openalex.org/I90610280"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanxiong Li","raw_affiliation_strings":["School of Electronic and Information Engineering, South China University of Technology, Guangzhou 510641, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Electronic and Information Engineering, South China University of Technology, Guangzhou 510641, China","institution_ids":["https://openalex.org/I90610280"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5101963530"],"corresponding_institution_ids":["https://openalex.org/I90610280"],"apc_list":{"value":2400,"currency":"CHF","value_usd":2598},"apc_paid":{"value":2400,"currency":"CHF","value_usd":2598},"fwci":1.0919,"has_fulltext":true,"cited_by_count":11,"citation_normalized_percentile":{"value":0.81042302,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"21","issue":"8","first_page":"2792","last_page":"2792"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/backup","display_name":"Backup","score":0.762527346611023},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7445622086524963},{"id":"https://openalex.org/keywords/divergence","display_name":"Divergence (linguistics)","score":0.7052746415138245},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.6767999529838562},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5794858932495117},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5442954897880554},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5396943092346191},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.44897475838661194},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.09896242618560791}],"concepts":[{"id":"https://openalex.org/C2780945871","wikidata":"https://www.wikidata.org/wiki/Q194274","display_name":"Backup","level":2,"score":0.762527346611023},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7445622086524963},{"id":"https://openalex.org/C207390915","wikidata":"https://www.wikidata.org/wiki/Q1230525","display_name":"Divergence (linguistics)","level":2,"score":0.7052746415138245},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6767999529838562},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5794858932495117},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5442954897880554},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5396943092346191},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.44897475838661194},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.09896242618560791},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.3390/s21082792","is_oa":true,"landing_page_url":"https://doi.org/10.3390/s21082792","pdf_url":"https://www.mdpi.com/1424-8220/21/8/2792/pdf?version=1619101797","source":{"id":"https://openalex.org/S101949793","display_name":"Sensors","issn_l":"1424-8220","issn":["1424-8220"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Sensors","raw_type":"journal-article"},{"id":"pmid:33921068","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/33921068","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Sensors (Basel, Switzerland)","raw_type":null},{"id":"pmh:oai:doaj.org/article:97a6502cc6c241f89be47666b80f7f62","is_oa":true,"landing_page_url":"https://doaj.org/article/97a6502cc6c241f89be47666b80f7f62","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Sensors, Vol 21, Iss 8, p 2792 (2021)","raw_type":"article"},{"id":"pmh:oai:mdpi.com:/1424-8220/21/8/2792/","is_oa":true,"landing_page_url":"https://dx.doi.org/10.3390/s21082792","pdf_url":null,"source":{"id":"https://openalex.org/S4306400947","display_name":"MDPI (MDPI AG)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210097602","host_organization_name":"Multidisciplinary Digital Publishing Institute (Switzerland)","host_organization_lineage":["https://openalex.org/I4210097602"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Sensors; Volume 21; Issue 8; Pages: 2792","raw_type":"Text"},{"id":"pmh:oai:pubmedcentral.nih.gov:8071405","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/8071405","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Sensors (Basel)","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.3390/s21082792","is_oa":true,"landing_page_url":"https://doi.org/10.3390/s21082792","pdf_url":"https://www.mdpi.com/1424-8220/21/8/2792/pdf?version=1619101797","source":{"id":"https://openalex.org/S101949793","display_name":"Sensors","issn_l":"1424-8220","issn":["1424-8220"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Sensors","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7400000095367432}],"awards":[{"id":"https://openalex.org/G243040613","display_name":"\u975e\u73b0\u573a\u8bf4\u8bdd\u4eba\u8ba4\u8bc1\u8bed\u97f3\u771f\u5b9e\u6027\u68c0\u6d4b\u5173\u952e\u6280\u672f\u7814\u7a76","funder_award_id":"61571192","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3716947235","display_name":null,"funder_award_id":"61571192, 61771200","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3994375257","display_name":"\u57fa\u4e8e\u6df1\u5ea6\u5b66\u4e60\u7684\u590d\u6742\u97f3\u9891\u8bf4\u8bdd\u4eba\u5206\u6790\u5173\u952e\u6280\u672f\u7814\u7a76","funder_award_id":"61771200","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3152578659.pdf","grobid_xml":"https://content.openalex.org/works/W3152578659.grobid-xml"},"referenced_works_count":56,"referenced_works":["https://openalex.org/W1690739335","https://openalex.org/W1821462560","https://openalex.org/W2145339207","https://openalex.org/W2194775991","https://openalex.org/W2549139847","https://openalex.org/W2561238782","https://openalex.org/W2592691248","https://openalex.org/W2620998106","https://openalex.org/W2765407302","https://openalex.org/W2796438033","https://openalex.org/W2803023299","https://openalex.org/W2806183416","https://openalex.org/W2810075754","https://openalex.org/W2883780447","https://openalex.org/W2883935097","https://openalex.org/W2895171208","https://openalex.org/W2904170036","https://openalex.org/W2934939260","https://openalex.org/W2937282529","https://openalex.org/W2944779197","https://openalex.org/W2945133121","https://openalex.org/W2945472816","https://openalex.org/W2946948417","https://openalex.org/W2953070460","https://openalex.org/W2956274063","https://openalex.org/W2962915600","https://openalex.org/W2963163009","https://openalex.org/W2963173418","https://openalex.org/W2964137095","https://openalex.org/W2964166828","https://openalex.org/W2976078745","https://openalex.org/W2979225039","https://openalex.org/W2982083293","https://openalex.org/W2984898826","https://openalex.org/W2987861506","https://openalex.org/W2987999870","https://openalex.org/W2994088087","https://openalex.org/W2996970889","https://openalex.org/W2997006708","https://openalex.org/W2997666887","https://openalex.org/W3004127093","https://openalex.org/W3015036596","https://openalex.org/W3032502780","https://openalex.org/W3034695001","https://openalex.org/W3036049548","https://openalex.org/W3036585823","https://openalex.org/W3049435445","https://openalex.org/W3097541226","https://openalex.org/W3118608800","https://openalex.org/W3123611094","https://openalex.org/W6740164494","https://openalex.org/W6751751081","https://openalex.org/W6757137021","https://openalex.org/W6762627229","https://openalex.org/W6768705926","https://openalex.org/W6770394654"],"related_works":["https://openalex.org/W2955195711","https://openalex.org/W2354454611","https://openalex.org/W2993266126","https://openalex.org/W2392283887","https://openalex.org/W2351388597","https://openalex.org/W2939925694","https://openalex.org/W2829881200","https://openalex.org/W4241986464","https://openalex.org/W2184647741","https://openalex.org/W1985525502"],"abstract_inverted_index":{"Knowledge":[0,115,208],"Distillation":[1,116],"(KD),":[2],"which":[3],"transfers":[4],"the":[5,53,64,71,87,93,97,100,105,120,135,139,149,156,166,179,190,194,202,207,214,243,252,259,265],"knowledge":[6,45],"from":[7],"a":[8,11,21,110,128,169,184],"teacher":[9,54,191,203],"to":[10,46,85,164,177,188,242],"student":[12],"network":[13],"by":[14,60,206],"penalizing":[15],"their":[16,77],"Kullback-Leibler":[17],"(KL)":[18],"divergence,":[19],"is":[20,57,175],"widely":[22],"used":[23],"tool":[24],"for":[25,79,89,183],"Deep":[26],"Neural":[27],"Network":[28,173],"(DNN)":[29],"compression":[30],"in":[31,55],"intelligent":[32],"sensor":[33],"systems.":[34],"Traditional":[35],"KD":[36,136],"uses":[37,119],"pre-trained":[38],"teacher,":[39],"while":[40],"self-KD":[41,56,73,112,130,246],"distills":[42],"its":[43,145,159],"own":[44],"achieve":[47],"better":[48,185],"performance.":[49],"The":[50,82],"role":[51],"of":[52,99,254],"usually":[58,260],"played":[59],"multi-branch":[61],"peers":[62],"or":[63],"identical":[65],"sample":[66],"with":[67,158,248],"different":[68,90,238],"augmentation.":[69],"However,":[70],"mentioned":[72],"methods":[74,247],"above":[75],"have":[76],"limitation":[78,106],"widespread":[80],"use.":[81],"former":[83],"needs":[84],"redesign":[86],"DNN":[88],"tasks,":[91],"and":[92,144,219,221,234],"latter":[94],"relies":[95],"on":[96,148,213,258],"effectiveness":[98,253],"augmentation":[101],"method.":[102,211],"To":[103],"avoid":[104],"above,":[107],"we":[108,126,200],"propose":[109,127],"new":[111],"method,":[113],"Memory-replay":[114],"(MrKD),":[117],"that":[118,133,228],"historical":[121,160,180,262],"models":[122,263],"as":[123,197],"teachers.":[124],"Firstly,":[125],"novel":[129],"training":[131,150,233,266],"method":[132],"penalizes":[134],"loss":[137],"between":[138],"current":[140],"model's":[141],"output":[142,161,182,205],"distributions":[143],"backup":[146],"outputs":[147,192],"trajectory.":[151,267],"This":[152],"strategy":[153],"can":[154],"regularize":[155],"model":[157,232],"distribution":[162],"space":[163],"stabilize":[165],"learning.":[167],"Secondly,":[168],"simple":[170],"Fully":[171],"Connected":[172],"(FCN)":[174],"applied":[176],"ensemble":[178],"teacher's":[181],"guidance.":[186],"Finally,":[187],"ensure":[189],"offer":[193],"right":[195],"class":[196],"ground":[198],"truth,":[199],"correct":[201],"logit":[204],"Adjustment":[209],"(KA)":[210],"Experiments":[212],"image":[215],"(dataset":[216,223],"CIFAR-100,":[217],"CIFAR-10,":[218],"CINIC-10)":[220],"audio":[222],"DCASE)":[224],"classification":[225],"tasks":[226],"show":[227],"MrKD":[229,255],"improves":[230],"single":[231],"working":[235],"efficiently":[236],"across":[237],"datasets.":[239],"In":[240],"contrast":[241],"existing":[244],"fancy":[245],"various":[249],"external":[250],"knowledge,":[251],"sheds":[256],"light":[257],"abandoned":[261],"during":[264]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":6},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}