{"id":"https://openalex.org/W4413358389","doi":"https://doi.org/10.1561/2200000116","title":"Continual Learning as Computationally Constrained Reinforcement Learning","display_name":"Continual Learning as Computationally Constrained Reinforcement Learning","publication_year":2025,"publication_date":"2025-08-20","ids":{"openalex":"https://openalex.org/W4413358389","doi":"https://doi.org/10.1561/2200000116"},"language":"en","primary_location":{"id":"doi:10.1561/2200000116","is_oa":false,"landing_page_url":"https://doi.org/10.1561/2200000116","pdf_url":null,"source":{"id":"https://openalex.org/S4210188176","display_name":"Foundations and Trends\u00ae in Machine Learning","issn_l":"1935-8237","issn":["1935-8237","1935-8245"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318575","host_organization_name":"Now Publishers","host_organization_lineage":["https://openalex.org/P4310318575"],"host_organization_lineage_names":["Now Publishers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Foundations and Trends\u00ae in Machine Learning","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5009018152","display_name":"Saurabh Kumar","orcid":"https://orcid.org/0000-0002-8383-6019"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saurabh Kumar","raw_affiliation_strings":["Stanford University Department of Computer Science, ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University Department of Computer Science, ,","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000779244","display_name":"Henrik Marklund","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Henrik Marklund","raw_affiliation_strings":["Stanford University Department of Computer Science, ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University Department of Computer Science, ,","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074050366","display_name":"Ashish Rao","orcid":"https://orcid.org/0000-0002-6676-8819"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ashish Rao","raw_affiliation_strings":["Stanford University Department of Computer Science, ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University Department of Computer Science, ,","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034415838","display_name":"Yifan Zhu","orcid":"https://orcid.org/0000-0002-7695-1633"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yifan Zhu","raw_affiliation_strings":["Stanford University Department of Electrical Engineering, ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University Department of Electrical Engineering, ,","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049973803","display_name":"Hong Jun Jeon","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hong Jun Jeon","raw_affiliation_strings":["Stanford University Department of Computer Science, ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University Department of Computer Science, ,","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101859711","display_name":"Yueyang Liu","orcid":"https://orcid.org/0000-0001-5894-8740"},"institutions":[{"id":"https://openalex.org/I2802678941","display_name":"Jones College","ror":"https://ror.org/05txzf778","country_code":"US","type":"education","lineage":["https://openalex.org/I2802678941"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liu Yueyang","raw_affiliation_strings":["Jones Graduate School of Business, Rice University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jones Graduate School of Business, Rice University ,","institution_ids":["https://openalex.org/I2802678941"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5045543562","display_name":"Benjamin Van Roy","orcid":"https://orcid.org/0000-0001-8364-3746"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Benjamin Van Roy","raw_affiliation_strings":["Stanford University Department of Electrical Engineering, ,","Stanford University Department of Management Science and Engineering, ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University Department of Electrical Engineering, ,","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Stanford University Department of Management Science and Engineering, ,","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.6879,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.87789127,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":"18","issue":"5","first_page":"913","last_page":"1053"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.6151000261306763,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.6151000261306763,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.5317999720573425,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7501935362815857},{"id":"https://openalex.org/keywords/reinforcement","display_name":"Reinforcement","score":0.5739498734474182},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5235499739646912},{"id":"https://openalex.org/keywords/error-driven-learning","display_name":"Error-driven learning","score":0.4619944393634796},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4256434738636017},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.19830283522605896},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.06508046388626099}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7501935362815857},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.5739498734474182},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5235499739646912},{"id":"https://openalex.org/C47932503","wikidata":"https://www.wikidata.org/wiki/Q5395689","display_name":"Error-driven learning","level":3,"score":0.4619944393634796},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4256434738636017},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.19830283522605896},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.06508046388626099}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1561/2200000116","is_oa":false,"landing_page_url":"https://doi.org/10.1561/2200000116","pdf_url":null,"source":{"id":"https://openalex.org/S4210188176","display_name":"Foundations and Trends\u00ae in Machine Learning","issn_l":"1935-8237","issn":["1935-8237","1935-8245"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318575","host_organization_name":"Now Publishers","host_organization_lineage":["https://openalex.org/P4310318575"],"host_organization_lineage_names":["Now Publishers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Foundations and Trends\u00ae in Machine Learning","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W1542791059","https://openalex.org/W1968250327","https://openalex.org/W1995875735","https://openalex.org/W2028138594","https://openalex.org/W2036963181","https://openalex.org/W2039522160","https://openalex.org/W2043195713","https://openalex.org/W2102115854","https://openalex.org/W2105934661","https://openalex.org/W2119885577","https://openalex.org/W2293212910","https://openalex.org/W2529894091","https://openalex.org/W2560647685","https://openalex.org/W2951777528","https://openalex.org/W2962821829","https://openalex.org/W3027406032","https://openalex.org/W3030314836","https://openalex.org/W3042443324","https://openalex.org/W3084695296","https://openalex.org/W3097816393","https://openalex.org/W3117215073","https://openalex.org/W3210638057","https://openalex.org/W4206530644","https://openalex.org/W4206547457","https://openalex.org/W4281697425","https://openalex.org/W4299401133","https://openalex.org/W4383982036"],"related_works":["https://openalex.org/W2371091044","https://openalex.org/W2171010636","https://openalex.org/W87513465","https://openalex.org/W2786230833","https://openalex.org/W1966803121","https://openalex.org/W2391666574","https://openalex.org/W3203256658","https://openalex.org/W2352650970","https://openalex.org/W1544514152","https://openalex.org/W1493952344"],"abstract_inverted_index":{"An":[0],"agent":[1,272],"that":[2,94,129,206,242,287],"accumulates":[3,273],"knowledge":[4],"to":[5,54,67,89,116,161,170,192,236,250,284,303],"develop":[6],"increasingly":[7],"sophisticated":[8],"skills":[9],"over":[10,228],"a":[11,29,50,61,125,171,180],"long":[12],"lifetime":[13],"could":[14],"advance":[15],"the":[16,35,69,157,163,188,199,211,240,245,275,304],"frontier":[17],"of":[18,24,37,46,63,71,165,183,201,247,262],"artificial":[19],"intelligence":[20],"capabilities.":[21],"The":[22],"design":[23,120],"such":[25,98,132,139],"agents,":[26,124],"which":[27,266,308],"remains":[28],"long-standing":[30],"challenge,":[31],"is":[32,179,190,213,218,230,255,286],"addressed":[33],"by":[34],"subject":[36,169],"continual":[38,47,85,144,158,279],"learning.":[39,77],"This":[40],"monograph":[41],"clarifies":[42],"and":[43,52,75,107,122,221],"formalizes":[44],"concepts":[45],"learning,":[48,176],"introducing":[49],"framework":[51],"tools":[53],"stimulate":[55],"further":[56],"research.":[57],"We":[58],"also":[59,300],"present":[60],"range":[62],"empirical":[64],"case":[65,182],"studies":[66],"illustrate":[68],"roles":[70],"forgetting,":[72,102],"relearning,":[73],"exploration,":[74],"auxiliary":[76],"Metrics":[78],"presented":[79],"in":[80,239,258,265,278,289,311],"previous":[81],"literature":[82],"for":[83,177],"evaluating":[84],"learning":[86,145,148,159,222,264,280],"agents":[87],"tend":[88],"focus":[90],"on":[91],"particular":[92],"behaviors":[93],"are":[95,205],"deemed":[96],"desirable,":[97],"as":[99,146,270],"avoiding":[100],"catastrophic":[101],"retaining":[103],"plasticity,":[104],"relearning":[105],"quickly,":[106],"maintaining":[108],"low":[109],"memory":[110],"or":[111,196],"compute":[112,151],"footprints.":[113],"In":[114,153],"order":[115],"systematically":[117],"reason":[118],"about":[119,223],"choices":[121],"compare":[123],"coherent,":[126],"holistic":[127],"objective":[128,160],"encompasses":[130],"all":[131,208],"requirements":[133],"would":[134],"be":[135,162,193],"helpful.":[136],"To":[137],"provide":[138],"an":[140,225,271],"objective,":[141],"we":[142,155,309],"cast":[143],"reinforcement":[147],"with":[149],"limited":[150],"resources.":[152],"particular,":[154],"pose":[156],"maximization":[164],"infinite-horizon":[166],"average":[167,203],"reward":[168,189,204],"computational":[172],"constraint.":[173],"Continual":[174],"supervised":[175],"example,":[178],"special":[181],"our":[184],"general":[185],"formulation":[186],"where":[187],"taken":[191],"negative":[194],"log-loss":[195],"accuracy.":[197],"Among":[198],"implications":[200],"maximizing":[202],"remembering":[207],"information":[209,217,248],"from":[210],"past":[212],"unnecessary,":[214],"forgetting":[215],"nonrecurring":[216],"not":[219],"\u201ccatastrophic,\u201d":[220],"how":[224],"environment":[226],"changes":[227],"time":[229],"useful.":[231],"Computational":[232],"constraints":[233,238,293,299],"give":[234,301],"rise":[235,302],"informational":[237,292],"sense":[241],"they":[243],"limit":[244],"amount":[246],"used":[249],"make":[251],"decisions.":[252],"A":[253],"consequence":[254],"that,":[256],"unlike":[257],"more":[259],"common":[260],"framings":[261],"machine":[263],"per-timestep":[267],"regret":[268,276],"vanishes":[269],"information,":[274],"experienced":[277],"typically":[281],"persists.":[282],"Related":[283],"this":[285],"even":[288],"stationary":[290],"environments,":[291],"can":[294],"incentivize":[295],"perpetual":[296],"adaptation.":[297],"Informational":[298],"familiar":[305],"stability-plasticity":[306],"dilemma,":[307],"formalize":[310],"information-theoretic":[312],"terms.":[313]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-06-19T17:40:00.097472","created_date":"2025-10-10T00:00:00"}