{"id":"https://openalex.org/W4200217066","doi":"https://doi.org/10.1109/drbsd754563.2021.00006","title":"Mitigating Catastrophic Forgetting in Deep Learning in a Streaming Setting Using Historical Summary","display_name":"Mitigating Catastrophic Forgetting in Deep Learning in a Streaming Setting Using Historical Summary","publication_year":2021,"publication_date":"2021-11-01","ids":{"openalex":"https://openalex.org/W4200217066","doi":"https://doi.org/10.1109/drbsd754563.2021.00006"},"language":"en","primary_location":{"id":"doi:10.1109/drbsd754563.2021.00006","doi":"https://doi.org/10.1109/drbsd754563.2021.00006","is_oa":false,"landing_page_url":"https://doi.org/10.1109/drbsd754563.2021.00006","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 7th International Workshop on Data Analysis and Reduction for Big Scientific Data (DRBSD-7)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.osti.gov/servlets/purl/1885239","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040970890","display_name":"Sajal Dash","orcid":"https://orcid.org/0000-0001-5308-914X","observed_orcids":["https://orcid.org/0000-0001-5308-914X"]},"institutions":[{"id":"https://openalex.org/I1289243028","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56","country_code":"US","type":"facility","lineage":["https://openalex.org/I1289243028","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210159294"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sajal Dash","raw_affiliation_strings":["National Center for Computational Sciences,Oak Ridge National Laboratory,Oak Ridge,TN,USA","Oak Ridge National Laboratory, National Center for Computational Sciences, Oak Ridge, TN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Center for Computational Sciences,Oak Ridge National Laboratory,Oak Ridge,TN,USA","institution_ids":["https://openalex.org/I1289243028"]},{"raw_affiliation_string":"Oak Ridge National Laboratory, National Center for Computational Sciences, Oak Ridge, TN, USA","institution_ids":["https://openalex.org/I1289243028"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051882926","display_name":"Junqi Yin","orcid":"https://orcid.org/0000-0003-3843-5520","observed_orcids":["https://orcid.org/0000-0003-3843-5520"]},"institutions":[{"id":"https://openalex.org/I1289243028","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56","country_code":"US","type":"facility","lineage":["https://openalex.org/I1289243028","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210159294"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Junqi Yin","raw_affiliation_strings":["National Center for Computational Sciences,Oak Ridge National Laboratory,Oak Ridge,TN,USA","Oak Ridge National Laboratory, National Center for Computational Sciences, Oak Ridge, TN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Center for Computational Sciences,Oak Ridge National Laboratory,Oak Ridge,TN,USA","institution_ids":["https://openalex.org/I1289243028"]},{"raw_affiliation_string":"Oak Ridge National Laboratory, National Center for Computational Sciences, Oak Ridge, TN, USA","institution_ids":["https://openalex.org/I1289243028"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035099577","display_name":"Mallikarjun Arjun Shankar","orcid":"https://orcid.org/0000-0001-5289-7460","observed_orcids":["https://orcid.org/0000-0001-5289-7460"]},"institutions":[{"id":"https://openalex.org/I1289243028","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56","country_code":"US","type":"facility","lineage":["https://openalex.org/I1289243028","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210159294"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mallikarjun Shankar","raw_affiliation_strings":["National Center for Computational Sciences,Oak Ridge National Laboratory,Oak Ridge,TN,USA","Oak Ridge National Laboratory, National Center for Computational Sciences, Oak Ridge, TN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Center for Computational Sciences,Oak Ridge National Laboratory,Oak Ridge,TN,USA","institution_ids":["https://openalex.org/I1289243028"]},{"raw_affiliation_string":"Oak Ridge National Laboratory, National Center for Computational Sciences, Oak Ridge, TN, USA","institution_ids":["https://openalex.org/I1289243028"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101916963","display_name":"Feiyi Wang","orcid":"https://orcid.org/0000-0002-0099-1559","observed_orcids":["https://orcid.org/0000-0002-0099-1559"]},"institutions":[{"id":"https://openalex.org/I1289243028","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56","country_code":"US","type":"facility","lineage":["https://openalex.org/I1289243028","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210159294"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Feiyi Wang","raw_affiliation_strings":["National Center for Computational Sciences,Oak Ridge National Laboratory,Oak Ridge,TN,USA","Oak Ridge National Laboratory, National Center for Computational Sciences, Oak Ridge, TN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Center for Computational Sciences,Oak Ridge National Laboratory,Oak Ridge,TN,USA","institution_ids":["https://openalex.org/I1289243028"]},{"raw_affiliation_string":"Oak Ridge National Laboratory, National Center for Computational Sciences, Oak Ridge, TN, USA","institution_ids":["https://openalex.org/I1289243028"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5058539554","display_name":"Wu-chun Feng","orcid":"https://orcid.org/0000-0002-6015-0727","observed_orcids":["https://orcid.org/0000-0002-6015-0727"]},"institutions":[{"id":"https://openalex.org/I859038795","display_name":"Virginia Tech","ror":"https://ror.org/02smfhw86","country_code":"US","type":"education","lineage":["https://openalex.org/I859038795"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wu-chun Feng","raw_affiliation_strings":["Virginia Tech,Department of Computer Science,Blacksburg,VA,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Virginia Tech,Department of Computer Science,Blacksburg,VA,USA","institution_ids":["https://openalex.org/I859038795"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.3465,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.57468422,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"11","last_page":"18"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.5878999829292297,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.5878999829292297,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.39399999380111694,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.0010999999940395355,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/catastrophic-forgetting","display_name":"catastrophic forgetting","score":1.0},{"id":"https://openalex.org/keywords/incremental-learning","display_name":"incremental learning","score":0.746999979019165},{"id":"https://openalex.org/keywords/data-streams","display_name":"data streams","score":0.7570000290870667},{"id":"https://openalex.org/keywords/historical-summary","display_name":"historical summary","score":0.9319999814033508},{"id":"https://openalex.org/keywords/data-summarization","display_name":"data summarization","score":0.9679999947547913},{"id":"https://openalex.org/keywords/random-sampling","display_name":"random sampling","score":0.8159999847412109},{"id":"https://openalex.org/keywords/micro-clustering","display_name":"micro-clustering","score":0.8880000114440918},{"id":"https://openalex.org/keywords/coresets","display_name":"coresets","score":0.9490000009536743},{"id":"https://openalex.org/keywords/autoencoder","display_name":"autoencoder","score":0.9369999766349792},{"id":"https://openalex.org/keywords/resnet-50","display_name":"ResNet50","score":0.6420000195503235},{"id":"https://openalex.org/keywords/language-model-training","display_name":"language model training","score":0.3889999985694885}],"concepts":[{"id":"https://openalex.org/C7149132","wikidata":"https://www.wikidata.org/wiki/Q1377840","display_name":"Forgetting","level":2,"score":0.8856505155563354},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8113901615142822},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.7019950747489929},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6955529451370239},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.5749183297157288},{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.5223963260650635},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5140095949172974},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5004823207855225},{"id":"https://openalex.org/C147168706","wikidata":"https://www.wikidata.org/wiki/Q1457734","display_name":"Recurrent neural network","level":3,"score":0.4886834919452667},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.40314340591430664},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09538742899894714},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/drbsd754563.2021.00006","doi":"https://doi.org/10.1109/drbsd754563.2021.00006","is_oa":false,"landing_page_url":"https://doi.org/10.1109/drbsd754563.2021.00006","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 7th International Workshop on Data Analysis and Reduction for Big Scientific Data (DRBSD-7)","raw_type":"proceedings-article"},{"id":"pmh:oai:osti.gov:1885239","doi":null,"is_oa":true,"landing_page_url":"https://www.osti.gov/biblio/1885239","pdf_url":"https://www.osti.gov/servlets/purl/1885239","source":{"id":"https://openalex.org/S4306402487","display_name":"OSTI OAI (U.S. Department of Energy Office of Scientific and Technical Information)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I139351228","host_organization_name":"Office of Scientific and Technical Information","host_organization_lineage":["https://openalex.org/I139351228"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:osti.gov:1885239","doi":null,"is_oa":true,"landing_page_url":"https://www.osti.gov/biblio/1885239","pdf_url":"https://www.osti.gov/servlets/purl/1885239","source":{"id":"https://openalex.org/S4306402487","display_name":"OSTI OAI (U.S. Department of Energy Office of Scientific and Technical Information)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I139351228","host_organization_name":"Office of Scientific and Technical Information","host_organization_lineage":["https://openalex.org/I139351228"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2596126017","display_name":null,"funder_award_id":"DE\u2013AC05\u2013000R22725","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"}],"funders":[{"id":"https://openalex.org/F4320306084","display_name":"U.S. Department of Energy","ror":"https://ror.org/01bj3aw27"},{"id":"https://openalex.org/F4320338287","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4200217066.pdf","grobid_xml":"https://content.openalex.org/works/W4200217066.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W1985602045","https://openalex.org/W1990517717","https://openalex.org/W2007339694","https://openalex.org/W2037979274","https://openalex.org/W2103753221","https://openalex.org/W2113839990","https://openalex.org/W2401932923","https://openalex.org/W2560647685","https://openalex.org/W2609002182","https://openalex.org/W2743151379","https://openalex.org/W2767677737","https://openalex.org/W2783989891","https://openalex.org/W2787998955","https://openalex.org/W2835056093","https://openalex.org/W2911327153","https://openalex.org/W2924448532","https://openalex.org/W2963264832","https://openalex.org/W2973226665"],"related_works":["https://openalex.org/W2791792504","https://openalex.org/W3046803548","https://openalex.org/W7212717898","https://openalex.org/W6945906690","https://openalex.org/W6960263419","https://openalex.org/W4381550994","https://openalex.org/W4377042032","https://openalex.org/W4402530380","https://openalex.org/W3123553256","https://openalex.org/W2903183390"],"abstract_inverted_index":{"Recent":[0],"advancements":[1],"in":[2,16,21,58,70,91,93,163,221,227],"scientific":[3],"equipment":[4],"and":[5,10,23,33,50,105,130,196,223],"the":[6,11,84,107,158,193,205,209,212,231,247,250,261,272],"adaptation":[7],"of":[8,13,160,211,269],"electronics":[9],"Internet":[12],"Things":[14],"(IoT)":[15],"our":[17,234],"everyday":[18],"lives":[19],"resulted":[20],"large":[22],"complex":[24],"data":[25,44,112,119],"production":[26],"at":[27,37],"a":[28,38,59,71,94,100,139,145,188,197],"high":[29],"rate.":[30],"Making":[31],"meaningful":[32],"timely":[34],"knowledge":[35],"discovery":[36],"modest":[39],"cost":[40],"from":[41],"this":[42],"big":[43],"is":[45,219,266],"difficult":[46],"for":[47,141,148,153,182],"computing":[48],"power":[49],"storage":[51],"limitations.":[52,68],"Training":[53],"deep":[54,150,174],"learning":[55,151,175],"models":[56,79,152],"incrementally":[57,77],"streaming":[60,95,154],"setting":[61],"can":[62],"help":[63],"us":[64],"with":[65,109,144,281],"overcoming":[66],"these":[67],"However,":[69],"well-known":[72],"phenomenon":[73],"named":[74],"catastrophic":[75,89,135,165,217,244,270],"forgetting,":[76,271],"trained":[78],"increasingly":[80],"perform":[81],"poorly":[82],"on":[83,185,192],"past":[85],"data.":[86,155],"To":[87],"mitigate":[88],"forgetting":[90,166,218],"training":[92,143,149,210,273],"setting,":[96],"we":[97,214],"propose":[98,117],"constructing":[99],"historical":[101,146,161,251,282],"summary":[102,108,147,162,252],"over":[103],"time":[104],"use":[106],"newly":[110],"arrived":[111],"during":[113],"incremental":[114,142],"training.":[115],"We":[116,137,156],"various":[118],"summarization":[120],"techniques":[121],"such":[122],"as":[123],"random":[124],"sampling,":[125],"micro":[126],"clustering,":[127],"coreset":[128],"computation,":[129],"Auto":[131],"Encoders":[132],"to":[133,203,238,243,258,279],"counteract":[134],"forgetting.":[136,245],"built":[138],"pipeline":[140],"demonstrate":[157],"effectiveness":[159],"mitigating":[164],"using":[167],"three":[168,172],"case":[169],"studies":[170],"involving":[171],"different":[173],"applications:":[176],"an":[177,228],"Artificial":[178],"Neural":[179,199],"Network":[180,200],"(ANN)":[181],"classification":[183,254],"task":[184],"MNIST":[186],"dataset,":[187,195],"language":[189],"model":[190],"(RNN-LM)":[191],"WikiText2":[194],"Convolutional":[198],"(CNN),":[201],"ResNet50":[202],"classify":[204],"ImageNet":[206],"dataset.":[207],"Through":[208],"models,":[213],"observe":[215],"that":[216],"evident":[220],"ANN":[222],"CNN":[224],"but":[225],"not":[226,267],"RNN.":[229],"For":[230,246,260],"first":[232],"task,":[233,249,263],"method":[235],"recovers":[236,253],"up":[237,257,278],"47.9%":[239],"lost":[240],"accuracy":[241,255],"due":[242],"third":[248],"by":[256,277],"25%.":[259],"second":[262],"though":[264],"there":[265],"proof":[268],"performance":[274],"(PPL)":[275],"improves":[276],"26%":[280],"summary.":[283]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2026-10-17T09:40:24.721904","created_date":"2021-12-31T00:00:00"}