{"id":"https://openalex.org/W3123553256","doi":null,"title":"Continual Learning and Forgetting in Deep Learning Models","display_name":"Continual Learning and Forgetting in Deep Learning Models","publication_year":2020,"publication_date":"2020-12-14","ids":{"openalex":"https://openalex.org/W3123553256","mag":"3123553256"},"language":"en","primary_location":{"id":"pmh:oai:uwspace.uwaterloo.ca:10012/16544","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/10012/16544","pdf_url":"http://hdl.handle.net/10012/16544","source":{"id":"https://openalex.org/S4306401661","display_name":"UWSpace (University of Waterloo)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I151746483","host_organization_name":"University of Waterloo","host_organization_lineage":["https://openalex.org/I151746483"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Doctoral Thesis"},"volume":null,"type":"dissertation","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://hdl.handle.net/10012/16544","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5056056673","display_name":"Alaa El Khatib","orcid":"https://orcid.org/0000-0003-3084-3589","observed_orcids":["https://orcid.org/0000-0003-3084-3589"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Alaa El Khatib","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-3084-3589","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5056056673"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/continual-learning","display_name":"continual learning","score":0.9290000200271606},{"id":"https://openalex.org/keywords/catastrophic-forgetting","display_name":"catastrophic forgetting","score":1.0},{"id":"https://openalex.org/keywords/auxiliary-tasks","display_name":"auxiliary tasks","score":0.6039999723434448},{"id":"https://openalex.org/keywords/regularization","display_name":"regularization","score":0.640999972820282},{"id":"https://openalex.org/keywords/replay-memory","display_name":"replay memory","score":0.9700000286102295},{"id":"https://openalex.org/keywords/image-classification","display_name":"image classification","score":0.335999995470047},{"id":"https://openalex.org/keywords/representation-learning","display_name":"representation learning","score":0.3240000009536743}],"concepts":[{"id":"https://openalex.org/C7149132","wikidata":"https://www.wikidata.org/wiki/Q1377840","display_name":"Forgetting","level":2,"score":0.6926677227020264},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5009903907775879},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47549518942832947},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4001419246196747},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3973376750946045},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3221004009246826}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:uwspace.uwaterloo.ca:10012/16544","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/10012/16544","pdf_url":"http://hdl.handle.net/10012/16544","source":{"id":"https://openalex.org/S4306401661","display_name":"UWSpace (University of Waterloo)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I151746483","host_organization_name":"University of Waterloo","host_organization_lineage":["https://openalex.org/I151746483"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Doctoral Thesis"},{"id":"mag:3123553256","doi":null,"is_oa":false,"landing_page_url":"https://uwspace.uwaterloo.ca/handle/10012/16544","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:uwspace.uwaterloo.ca:10012/16544","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/10012/16544","pdf_url":"http://hdl.handle.net/10012/16544","source":{"id":"https://openalex.org/S4306401661","display_name":"UWSpace (University of Waterloo)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I151746483","host_organization_name":"University of Waterloo","host_organization_lineage":["https://openalex.org/I151746483"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Doctoral Thesis"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3123553256.pdf","grobid_xml":"https://content.openalex.org/works/W3123553256.grobid-xml"},"referenced_works_count":57,"referenced_works":["https://openalex.org/W104574262","https://openalex.org/W854541894","https://openalex.org/W1498436455","https://openalex.org/W1503398984","https://openalex.org/W1682403713","https://openalex.org/W1821462560","https://openalex.org/W2036963181","https://openalex.org/W2052089281","https://openalex.org/W2060277733","https://openalex.org/W2076118331","https://openalex.org/W2099471712","https://openalex.org/W2100602534","https://openalex.org/W2110048843","https://openalex.org/W2112796928","https://openalex.org/W2116522068","https://openalex.org/W2118858186","https://openalex.org/W2147800946","https://openalex.org/W2164724958","https://openalex.org/W2304648132","https://openalex.org/W2335728318","https://openalex.org/W2411541852","https://openalex.org/W2415865124","https://openalex.org/W2426267443","https://openalex.org/W2473930607","https://openalex.org/W2560647685","https://openalex.org/W2565989828","https://openalex.org/W2618767506","https://openalex.org/W2731634768","https://openalex.org/W2737492962","https://openalex.org/W2740678877","https://openalex.org/W2744654301","https://openalex.org/W2747206364","https://openalex.org/W2750384547","https://openalex.org/W2770882886","https://openalex.org/W2774616426","https://openalex.org/W2786446225","https://openalex.org/W2791091755","https://openalex.org/W2792568620","https://openalex.org/W2902986194","https://openalex.org/W2949117887","https://openalex.org/W2962724315","https://openalex.org/W2963178577","https://openalex.org/W2963606038","https://openalex.org/W2963813679","https://openalex.org/W2963850662","https://openalex.org/W2963872552","https://openalex.org/W2964121744","https://openalex.org/W2964308564","https://openalex.org/W2967588839","https://openalex.org/W2971176100","https://openalex.org/W2977479414","https://openalex.org/W2980753060","https://openalex.org/W3009987980","https://openalex.org/W3014562431","https://openalex.org/W3035546118","https://openalex.org/W3118608800","https://openalex.org/W4319988532"],"related_works":["https://openalex.org/W2791792504","https://openalex.org/W7078395502","https://openalex.org/W2913006822","https://openalex.org/W6960263419","https://openalex.org/W4402530380","https://openalex.org/W2811346170","https://openalex.org/W7129041366","https://openalex.org/W7220986610","https://openalex.org/W3211394631","https://openalex.org/W3098061682"],"abstract_inverted_index":{"Continual":[0],"learning":[1,6,22,62,87,97,163,301,335,344,378,426,477,509,538],"is":[2,102,188,237,337,392,461],"a":[3,25,73,191,198,207,214,233,238,304,329,449],"framework":[4],"of":[5,16,20,47,55,85,113,149,160,179,232,241,256,290,328,365,411,416,424,457,475,527],"in":[7,60,140,182,332,414,446,479,507,517,540],"which":[8,108],"we":[9,146,174,261,320,361,407,443,471],"aim":[10,187,267],"to":[11,35,110,116,137,189,201,268,275,287,306,324,338,356,463,488,514],"move":[12],"beyond":[13],"the":[14,68,77,83,100,103,111,161,168,176,186,230,251,299,307,311,325,333,340,363,390,409,421,437,455,458,473,480,490,493,508,518,528,541],"limitations":[15],"standard":[17],"isolated":[18],"optimization":[19,308,341],"deep":[21,86],"models":[23,30,432,478],"toward":[24,343],"more":[26,345],"intelligent":[27],"setting,":[28],"where":[29,185],"or":[31],"agents":[32],"are":[33,126,353,433,440],"able":[34],"accumulate":[36],"skills":[37,119],"and":[38,43,65,70,120,152,156,196,217,244,348,367,469,483,486,531],"knowledge,":[39],"across":[40],"diverse":[41],"tasks":[42,370,419,439],"over":[44,67,76,99],"extended":[45],"periods":[46],"time,":[48],"much":[49,54],"like":[50],"humans":[51],"do.":[52],"Like":[53],"neural":[56,114,141],"networks":[57,115],"research,":[58],"interest":[59],"continual":[61,96,162,300,334,425,476,537],"has":[63,94],"ebbed":[64],"flowed":[66],"decades,":[69],"ultimately":[71,285],"saw":[72],"sharp":[74],"increase":[75],"past":[78],"few":[79],"years,":[80],"buoyed":[81],"by":[82,247,271],"successes":[84],"thus":[88,349],"far.":[89],"\\n":[90,165,172,259,405,467,521],"\\nOne":[91],"obstacle":[92],"that":[93,227,229,263,266,284,295,322,352,359,380,389,430,498],"dominated":[95],"research":[98],"years":[101],"so-called":[104,481],"catastrophic":[105],"forgetting":[106,139,195,270],"phenomenon,":[107],"refers":[109],"tendency":[112],"\"forget\"":[117],"older":[118],"knowledge":[121],"as":[122,124,280,371],"soon":[123],"they":[125],"subsequently":[127],"optimized":[128],"for":[129],"additional":[130,203],"tasks.":[131,204],"Researchers":[132],"have":[133],"proposed":[134,221,529],"various":[135],"approaches":[136,151,487,530],"counter":[138],"networks.":[142],"In":[143,318],"this":[144],"dissertation,":[145],"review":[147],"some":[148],"those":[150,447],"build":[153],"upon":[154],"them,":[155],"address":[157,175],"other":[158],"aspects":[159],"problem.":[164],"\\nWe":[166,522],"make":[167],"following":[169],"four":[170],"contributions.":[171],"\\nFirst,":[173],"critical":[177],"role":[178],"importance":[180,210],"estimation":[181,211],"fixed-capacity":[183],"models,":[184],"find":[190],"balance":[192],"between":[193,253,492],"countering":[194],"preserving":[197],"model's":[199,330],"capacity":[200,331],"learn":[202],"We":[205,293,387,428,496],"propose":[206],"novel":[208],"unit":[209],"approach,":[212],"with":[213],"small":[215,450],"memory":[216],"computational":[218],"footprint.":[219],"The":[220],"approach":[222,309,391],"builds":[223],"on":[224,420],"recent":[225,536],"work":[226],"showed":[228],"average":[231],"unit's":[234],"activation":[235,257],"values":[236],"good":[239],"indicator":[240],"its":[242],"importance,":[243],"extends":[245],"it":[246,399],"taking":[248,296],"into":[249,297],"consideration":[250],"separation":[252],"class-conditional":[254],"distributions":[255],"values.":[258],"\\nSecond,":[260],"observe":[262],"most":[264],"methods":[265,539],"prevent":[269],"explicitly":[272,402],"penalizing":[273],"changes":[274],"parameters":[276],"can":[277,511],"be":[278,512],"seen":[279],"post":[281],"hoc":[282],"remedies":[283],"lead":[286],"inefficient":[288],"use":[289,327,364],"model":[291],"capacity.":[292],"argue":[294,321],"account":[298],"objective":[302],"requires":[303],"modification":[305],"from":[310,504],"start":[312],"rather":[313],"than":[314],"only":[315],"after":[316],"learning.":[317],"particular,":[319],"key":[323],"effective":[326],"setting":[336],"drive":[339],"process":[342],"general,":[346],"reusable,":[347],"durable":[350],"representations":[351,379],"less":[354],"susceptible":[355],"forgetting.":[357,404],"To":[358],"end,":[360],"explore":[362,408,472],"supervised":[366],"unsupervised":[368],"auxiliary":[369,500],"regularization,":[372],"not":[373,401,502],"against":[374,377,535],"forgetting,":[375,396],"but":[376],"narrowly":[381],"target":[382],"any":[383,505],"single":[384],"classification":[385,418],"task.":[386],"show":[388,429,444,497],"successful":[393],"at":[394],"mitigating":[395],"even":[397,453],"though":[398],"does":[400],"penalize":[403],"\\nThird,":[406],"effect":[410],"inter-task":[412],"similarity":[413],"sequences":[415],"image":[417],"overall":[422],"performance":[423,474,516,534],"models.":[427],"certain":[431],"adversely":[434],"affected":[435],"when":[436],"learned":[438],"dissimilar.":[441],"Moreover,":[442],"that,":[445],"cases,":[448],"replay":[451],"memory,":[452],"1%":[454],"size":[456],"training":[459],"data,":[460,501],"enough":[462],"significantly":[464],"improve":[465,515],"performance.":[466],"\\nFourth":[468],"lastly,":[470],"multi-head":[482],"single-head":[484,519],"settings":[485],"narrow":[489],"gap":[491],"two":[494],"settings.":[495],"unlabelled":[499],"sampled":[503],"task":[506],"sequence,":[510],"used":[513],"setting.":[520],"provide":[523],"extensive":[524],"empirical":[525],"evaluation":[526],"compare":[532],"their":[533],"literature.":[542]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2021-02-01T00:00:00"}