{"id":"https://openalex.org/W3003289092","doi":null,"title":"Does an LSTM forget more than a CNN? An empirical study of catastrophic forgetting in NLP","display_name":"Does an LSTM forget more than a CNN? An empirical study of catastrophic forgetting in NLP","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W3003289092","mag":"3003289092"},"language":"en","primary_location":{"id":"mag:3003289092","doi":null,"is_oa":false,"landing_page_url":"https://www.aclweb.org/anthology/U19-1011.pdf","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037240854","display_name":"Gaurav S. Arora","orcid":"https://orcid.org/0000-0002-3615-1770","observed_orcids":["https://orcid.org/0000-0002-3615-1770"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gaurav Arora","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-3615-1770","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082351931","display_name":"Afshin Rahimi","orcid":"https://orcid.org/0000-0001-7889-5025","observed_orcids":["https://orcid.org/0000-0001-7889-5025"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Afshin Rahimi","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-7889-5025","affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5103085805","display_name":"Timothy Baldwin","orcid":"https://orcid.org/0000-0002-4445-1386","observed_orcids":["https://orcid.org/0000-0002-4445-1386","https://orcid.org/0000-0003-4525-6950"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Timothy Baldwin","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-4445-1386","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.3359,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.83962629,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"77","last_page":"86"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0071000000461936,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/catastrophic-forgetting","display_name":"catastrophic forgetting","score":1.0},{"id":"https://openalex.org/keywords/continual-learning","display_name":"continual learning","score":0.8180000185966492},{"id":"https://openalex.org/keywords/transfer-learning","display_name":"transfer learning","score":0.6660000085830688},{"id":"https://openalex.org/keywords/lstm","display_name":"LSTM","score":0.7599999904632568},{"id":"https://openalex.org/keywords/convolutional-neural-networks","display_name":"convolutional neural networks","score":0.9509999752044678},{"id":"https://openalex.org/keywords/max-pooling","display_name":"max pooling","score":0.8989999890327454},{"id":"https://openalex.org/keywords/curriculum-learning","display_name":"curriculum learning","score":0.9919999837875366},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"fine-tuning","score":0.8529999852180481},{"id":"https://openalex.org/keywords/contextualized-embeddings","display_name":"contextual embeddings","score":0.9739999771118164},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"feature extraction","score":0.5559999942779541},{"id":"https://openalex.org/keywords/hyperparameter-effects","display_name":"hyperparameter effects","score":0.5189999938011169}],"concepts":[{"id":"https://openalex.org/C7149132","wikidata":"https://www.wikidata.org/wiki/Q1377840","display_name":"Forgetting","level":2,"score":0.9753319025039673},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7480544447898865},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6583318710327148},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6567206382751465},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.5561283826828003},{"id":"https://openalex.org/C117978034","wikidata":"https://www.wikidata.org/wiki/Q5422192","display_name":"Extractor","level":2,"score":0.5071086883544922},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.48958620429039},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.4362568259239197},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39774879813194275},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3294692635536194},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.08876413106918335},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07104349136352539},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.06564417481422424},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C21880701","wikidata":"https://www.wikidata.org/wiki/Q2144042","display_name":"Process engineering","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:3003289092","doi":null,"is_oa":false,"landing_page_url":"https://www.aclweb.org/anthology/U19-1011.pdf","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W1682403713","https://openalex.org/W1821462560","https://openalex.org/W2036963181","https://openalex.org/W2099049980","https://openalex.org/W2109553965","https://openalex.org/W2114524997","https://openalex.org/W2163605009","https://openalex.org/W2164724958","https://openalex.org/W2251939518","https://openalex.org/W2296073425","https://openalex.org/W2473930607","https://openalex.org/W2554863749","https://openalex.org/W2583761661","https://openalex.org/W2770882886","https://openalex.org/W2787560479","https://openalex.org/W2793978524","https://openalex.org/W2798812533","https://openalex.org/W2806120502","https://openalex.org/W2891177506","https://openalex.org/W2911681509","https://openalex.org/W2921848006","https://openalex.org/W2962724315","https://openalex.org/W2963341956","https://openalex.org/W2963813679","https://openalex.org/W2964352358","https://openalex.org/W2964790801","https://openalex.org/W2989499211"],"related_works":["https://openalex.org/W4378768767","https://openalex.org/W2972566450","https://openalex.org/W7125773839","https://openalex.org/W7139924562","https://openalex.org/W4404010374","https://openalex.org/W7220977103","https://openalex.org/W3005863213","https://openalex.org/W7128211987","https://openalex.org/W7105671333","https://openalex.org/W3106286430"],"abstract_inverted_index":{"Catastrophic":[0],"forgetting":[1,59,73,99,131],"\u2014":[2,29],"whereby":[3],"a":[4,13,20,31,61,110],"model":[5],"trained":[6],"on":[7,12,129],"one":[8],"task":[9,28,112,117],"is":[10,30,80,91,140],"fine-tuned":[11],"second,":[14],"and":[15,56,132],"in":[16,23,33,44,60,144],"doing":[17],"so,":[18],"suffers":[19],"\u201ccatastrophic\u201d":[21],"drop":[22],"performance":[24],"over":[25],"the":[26,34,92,114,123],"first":[27],"hurdle":[32],"development":[35],"of":[36,52,116,125],"better":[37],"transfer":[38],"learning":[39,146],"techniques.":[40],"Despite":[41],"impressive":[42],"progress":[43],"reducing":[45],"catastrophic":[46,130],"forgetting,":[47],"we":[48,66],"have":[49],"limited":[50],"understanding":[51],"how":[53],"different":[54],"architectures":[55],"hyper-parameters":[57],"affect":[58],"network.":[62],"With":[63],"this":[64],"study,":[65],"aim":[67],"to":[68,101,142],"understand":[69],"factors":[70],"which":[71,95],"cause":[72],"during":[74],"sequential":[75],"training.":[76],"Our":[77],"primary":[78],"finding":[79],"that":[81,89,106,134],"CNNs":[82,97],"forget":[83],"less":[84],"than":[85],"LSTMs.":[86,102],"We":[87,103,121],"show":[88],"max-pooling":[90],"underlying":[93],"operation":[94],"helps":[96],"alleviate":[98],"compared":[100],"also":[104],"found":[105,133],"curriculum":[107],"learning,":[108],"placing":[109],"hard":[111],"towards":[113],"end":[115],"sequence,":[118],"reduces":[119],"forgetting.":[120],"analysed":[122],"effect":[124],"fine-tuning":[126,143],"contextual":[127],"embeddings":[128,136],"using":[135],"as":[137],"feature":[138],"extractor":[139],"preferable":[141],"continual":[145],"setup.":[147]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":3}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2020-02-07T00:00:00"}