{"id":"https://openalex.org/W2803681939","doi":"https://doi.org/10.24963/ijcai.2019/355","title":"Nostalgic Adam: Weighting More of the Past Gradients When Designing the Adaptive Learning Rate","display_name":"Nostalgic Adam: Weighting More of the Past Gradients When Designing the Adaptive Learning Rate","publication_year":2019,"publication_date":"2019-07-28","ids":{"openalex":"https://openalex.org/W2803681939","doi":"https://doi.org/10.24963/ijcai.2019/355","mag":"2803681939"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2019/355","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2019/355","pdf_url":"https://www.ijcai.org/proceedings/2019/0355.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Eighth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.ijcai.org/proceedings/2019/0355.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067680401","display_name":"Haiwen Huang","orcid":null},"institutions":[{"id":"https://openalex.org/I111483173","display_name":"King University","ror":"https://ror.org/01evb6z23","country_code":"US","type":"education","lineage":["https://openalex.org/I111483173"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210096250","display_name":"Beijing Institute of Big Data Research","ror":"https://ror.org/00s1sz824","country_code":"CN","type":"facility","lineage":["https://openalex.org/I20231570","https://openalex.org/I37796252","https://openalex.org/I4210096250"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Haiwen Huang","raw_affiliation_strings":["School of Mathematical Sciences, Peking University","Center for Data Science, Peking University Beijing Institute of Big Data Research Beijing, China","School of Mathematical Sciences Peking University, Beijing, 100871","Beijing International Center for Mathematical Research, Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Mathematical Sciences, Peking University","institution_ids":["https://openalex.org/I111483173","https://openalex.org/I20231570"]},{"raw_affiliation_string":"Center for Data Science, Peking University Beijing Institute of Big Data Research Beijing, China","institution_ids":["https://openalex.org/I4210096250","https://openalex.org/I20231570"]},{"raw_affiliation_string":"School of Mathematical Sciences Peking University, Beijing, 100871","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Beijing International Center for Mathematical Research, Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100371648","display_name":"Chang Wang","orcid":"https://orcid.org/0000-0002-6864-7877"},"institutions":[{"id":"https://openalex.org/I111483173","display_name":"King University","ror":"https://ror.org/01evb6z23","country_code":"US","type":"education","lineage":["https://openalex.org/I111483173"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210096250","display_name":"Beijing Institute of Big Data Research","ror":"https://ror.org/00s1sz824","country_code":"CN","type":"facility","lineage":["https://openalex.org/I20231570","https://openalex.org/I37796252","https://openalex.org/I4210096250"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Chang Wang","raw_affiliation_strings":["School of Mathematical Sciences, Peking University","Center for Data Science, Peking University Beijing Institute of Big Data Research Beijing, China","Beijing International Center for Mathematical Research, Peking University","School of Mathematical Sciences Peking University, Beijing, 100871"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Mathematical Sciences, Peking University","institution_ids":["https://openalex.org/I111483173","https://openalex.org/I20231570"]},{"raw_affiliation_string":"Center for Data Science, Peking University Beijing Institute of Big Data Research Beijing, China","institution_ids":["https://openalex.org/I4210096250","https://openalex.org/I20231570"]},{"raw_affiliation_string":"Beijing International Center for Mathematical Research, Peking University","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"School of Mathematical Sciences Peking University, Beijing, 100871","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5035683291","display_name":"Bin Dong","orcid":"https://orcid.org/0000-0003-1295-3362"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210096250","display_name":"Beijing Institute of Big Data Research","ror":"https://ror.org/00s1sz824","country_code":"CN","type":"facility","lineage":["https://openalex.org/I20231570","https://openalex.org/I37796252","https://openalex.org/I4210096250"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Bin Dong","raw_affiliation_strings":["Beijing International Center for Mathematical Research, Peking University","Center for Data Science, Peking University","Laboratory for Biomedical Image Analysis, Beijing Institute of Big Data Research, Beijing","Center for Data Science, Peking University Beijing Institute of Big Data Research Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing International Center for Mathematical Research, Peking University","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Center for Data Science, Peking University","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Laboratory for Biomedical Image Analysis, Beijing Institute of Big Data Research, Beijing","institution_ids":["https://openalex.org/I4210096250"]},{"raw_affiliation_string":"Center for Data Science, Peking University Beijing Institute of Big Data Research Beijing, China","institution_ids":["https://openalex.org/I4210096250","https://openalex.org/I20231570"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5035683291"],"corresponding_institution_ids":["https://openalex.org/I20231570","https://openalex.org/I4210096250"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":64,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2556","last_page":"2562"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10100","display_name":"Metaheuristic Optimization Algorithms Research","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.7031487226486206},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7028456330299377},{"id":"https://openalex.org/keywords/weighting","display_name":"Weighting","score":0.698650598526001},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5952929258346558},{"id":"https://openalex.org/keywords/mathematical-proof","display_name":"Mathematical proof","score":0.5878860354423523},{"id":"https://openalex.org/keywords/divergence","display_name":"Divergence (linguistics)","score":0.5875729322433472},{"id":"https://openalex.org/keywords/rate-of-convergence","display_name":"Rate of convergence","score":0.5771511793136597},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.49585995078086853},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.43959322571754456},{"id":"https://openalex.org/keywords/order","display_name":"Order (exchange)","score":0.4100610017776489},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.3873148560523987},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.36794161796569824},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.29270052909851074},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.17952945828437805},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.11309042572975159}],"concepts":[{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.7031487226486206},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7028456330299377},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.698650598526001},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5952929258346558},{"id":"https://openalex.org/C108710211","wikidata":"https://www.wikidata.org/wiki/Q11538","display_name":"Mathematical proof","level":2,"score":0.5878860354423523},{"id":"https://openalex.org/C207390915","wikidata":"https://www.wikidata.org/wiki/Q1230525","display_name":"Divergence (linguistics)","level":2,"score":0.5875729322433472},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.5771511793136597},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49585995078086853},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.43959322571754456},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.4100610017776489},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.3873148560523987},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36794161796569824},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.29270052909851074},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.17952945828437805},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.11309042572975159},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C126838900","wikidata":"https://www.wikidata.org/wiki/Q77604","display_name":"Radiology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.24963/ijcai.2019/355","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2019/355","pdf_url":"https://www.ijcai.org/proceedings/2019/0355.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Eighth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1805.07557","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1805.07557","pdf_url":"https://arxiv.org/pdf/1805.07557","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.24963/ijcai.2019/355","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2019/355","pdf_url":"https://www.ijcai.org/proceedings/2019/0355.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Eighth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320324787","display_name":"Peking University","ror":"https://ror.org/02v51f717"},{"id":"https://openalex.org/F4320337380","display_name":"Division of Mathematical Sciences","ror":"https://ror.org/051fftw81"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2803681939.pdf","grobid_xml":"https://content.openalex.org/works/W2803681939.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W6908809","https://openalex.org/W1522301498","https://openalex.org/W1988720110","https://openalex.org/W2146502635","https://openalex.org/W2148825261","https://openalex.org/W2194775991","https://openalex.org/W2405601855","https://openalex.org/W2619516334","https://openalex.org/W2776855315","https://openalex.org/W2777662428","https://openalex.org/W2785523195","https://openalex.org/W2785940809","https://openalex.org/W2808042107","https://openalex.org/W2892559787","https://openalex.org/W2962933129","https://openalex.org/W2963208657","https://openalex.org/W2963562522","https://openalex.org/W2964121744","https://openalex.org/W2964137095","https://openalex.org/W2969945254"],"related_works":["https://openalex.org/W2180954594","https://openalex.org/W2052835778","https://openalex.org/W4394650907","https://openalex.org/W2049003611","https://openalex.org/W2108418243","https://openalex.org/W2127804977","https://openalex.org/W164103134","https://openalex.org/W4254119641","https://openalex.org/W3196207352","https://openalex.org/W155536188"],"abstract_inverted_index":{"First-order":[0],"optimization":[1],"algorithms":[2,13],"have":[3,25],"been":[4],"proven":[5],"prominent":[6],"in":[7,33,100],"deep":[8],"learning.":[9],"In":[10,45],"particu-":[11],"lar,":[12],"such":[14],"as":[15,91],"RMSProp":[16],"and":[17,41,129],"Adam":[18,75,99],"are":[19,52,133],"extremely":[20],"popular.":[21],"However,":[22],"recent":[23,104],"works":[24],"pointed":[26],"out":[27],"the":[28,58,63,73,82,95,103],"lack":[29],"of":[30,54,57,98,106],"\u201clong-term":[31],"memory\u201d":[32],"Adam-like":[34],"algorithms,":[35],"which":[36],"could":[37],"hamper":[38],"their":[39],"performance":[40],"lead":[42],"to":[43,94,102,124],"divergence.":[44],"our":[46],"study,":[47],"we":[48],"observe":[49],"that":[50,116],"there":[51],"benefits":[53],"weighting":[55],"more":[56],"past":[59],"gradients":[60],"when":[61],"designing":[62],"adaptive":[64],"learning":[65],"rate.":[66,86],"We":[67],"therefore":[68],"propose":[69],"an":[70],"algorithm":[71],"called":[72],"Nostalgic":[74],"(NosAdam)":[76],"with":[77],"theoretically":[78],"guaranteed":[79],"convergence":[80,85],"at":[81],"best":[83],"known":[84],"NosAdam":[87,117],"can":[88],"be":[89],"regarded":[90],"a":[92,119],"fix":[93],"non-convergence":[96],"issue":[97],"alternative":[101,121],"work":[105],"[Reddi":[107],"et":[108],"al.,":[109],"2018].":[110],"Our":[111],"preliminary":[112],"numerical":[113],"experiments":[114],"show":[115],"is":[118],"promising":[120],"al-":[122],"gorithm":[123],"Adam.":[125],"The":[126],"proofs,":[127],"code":[128],"other":[130],"supplementary":[131],"materials":[132],"already":[134],"released.":[135]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":11},{"year":2020,"cited_by_count":16},{"year":2019,"cited_by_count":7},{"year":2018,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}