{"id":"https://openalex.org/W4407185734","doi":"https://doi.org/10.52202/085713-5215","title":"How Memory in Optimization Algorithms Implicitly Modifies the Loss","display_name":"How Memory in Optimization Algorithms Implicitly Modifies the Loss","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4407185734","doi":"https://doi.org/10.52202/085713-5215"},"language":"en","primary_location":{"id":"doi:10.52202/085713-5215","doi":"https://doi.org/10.52202/085713-5215","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-5215","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.02132","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5018060531","display_name":"Matias D. Cattaneo","orcid":"https://orcid.org/0000-0003-0493-7506","observed_orcids":["https://orcid.org/0000-0003-0493-7506"]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matias Cattaneo","raw_affiliation_strings":["Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5124872842","display_name":"Boris Shigida","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Boris Shigida","raw_affiliation_strings":["Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton University","institution_ids":["https://openalex.org/I20089843"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I20089843"],"apc_list":null,"apc_paid":null,"fwci":1.472,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.84740236,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"173262","last_page":"173299"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10963","display_name":"Advanced Optimization Algorithms Research","score":0.0,"subfield":{"id":"https://openalex.org/subfields/2612","display_name":"Numerical Analysis"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/implicit-memory","display_name":"implicit memory","score":0.6029999852180481},{"id":"https://openalex.org/keywords/optimization-algorithms","display_name":"optimization algorithms","score":0.5189999938011169},{"id":"https://openalex.org/keywords/gradient-descent-with-momentum","display_name":"gradient descent with momentum","score":0.7260000109672546},{"id":"https://openalex.org/keywords/lion-optimizer","display_name":"Lion optimizer","score":0.6600000262260437},{"id":"https://openalex.org/keywords/adamw","display_name":"AdamW","score":0.925000011920929},{"id":"https://openalex.org/keywords/implicit-regularization","display_name":"implicit regularization","score":0.6449999809265137},{"id":"https://openalex.org/keywords/generalization","display_name":"generalization","score":0.7950000166893005},{"id":"https://openalex.org/keywords/deep-learning-optimization","display_name":"deep learning optimization","score":0.7179999947547913}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5662040710449219},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4876464605331421},{"id":"https://openalex.org/C2987595161","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Optimization algorithm","level":2,"score":0.44776543974876404},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.34178096055984497},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2470451295375824},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2060135006904602}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.52202/085713-5215","doi":"https://doi.org/10.52202/085713-5215","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-5215","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2502.02132","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.02132","pdf_url":"https://arxiv.org/pdf/2502.02132","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.02132","doi":"https://doi.org/10.48550/arxiv.2502.02132","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.02132","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.02132","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.02132","pdf_url":"https://arxiv.org/pdf/2502.02132","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G4625221359","display_name":"Nonparametric Estimation and Inference with Network Data","funder_award_id":"2210561","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G78141540","display_name":"Partitioning-Based Learning Methods for Treatment Effect Estimation and Inference","funder_award_id":"2241575","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4407185734.pdf","grobid_xml":"https://content.openalex.org/works/W4407185734.grobid-xml"},"referenced_works_count":14,"referenced_works":["https://openalex.org/W1988720110","https://openalex.org/W2790253170","https://openalex.org/W2945236893","https://openalex.org/W2946578452","https://openalex.org/W2950220847","https://openalex.org/W2964121744","https://openalex.org/W2980149079","https://openalex.org/W3103334733","https://openalex.org/W4289446290","https://openalex.org/W4367553283","https://openalex.org/W4386793277","https://openalex.org/W4403781043","https://openalex.org/W4417070075","https://openalex.org/W4417082077"],"related_works":["https://openalex.org/W4414972817","https://openalex.org/W4392182272","https://openalex.org/W3174530962","https://openalex.org/W7220692321","https://openalex.org/W7161799498","https://openalex.org/W3207370098","https://openalex.org/W7165473754","https://openalex.org/W7203866164","https://openalex.org/W4399795545","https://openalex.org/W7164865171"],"abstract_inverted_index":{"In":[0],"modern":[1],"optimization":[2,63,123],"methods":[3],"used":[4],"in":[5,75],"deep":[6],"learning,":[7],"each":[8],"update":[9,77],"depends":[10],"on":[11],"the":[12,28,33,76,79,95,108,111,122,138],"history":[13],"of":[14,94,107,113,128,140],"previous":[15],"iterations,":[16],"often":[17],"referred":[18],"to":[19],"as":[20,27,104],"memory,":[21],"and":[22,82,110],"this":[23,114],"dependence":[24],"decays":[25],"fast":[26],"iterates":[29,74],"go":[30],"further":[31],"into":[32],"past.":[34],"For":[35],"example,":[36],"gradient":[37],"descent":[38],"with":[39,65],"momentum":[40],"has":[41],"exponentially":[42,46],"decaying":[43],"memory":[44,90,119,145],"through":[45],"averaged":[47],"past":[48,73],"gradients.":[49],"We":[50],"introduce":[51],"a":[52,57,85,92,105,150],"general":[53],"technique":[54],"for":[55,153],"identifying":[56],"memoryless":[58],"algorithm":[59,64],"that":[60,133,146],"approximates":[61],"an":[62,126],"memory.":[66],"It":[67],"is":[68],"obtained":[69],"by":[70,78,144],"replacing":[71],"all":[72],"current":[80,96],"one,":[81],"then":[83],"adding":[84],"correction":[86,99],"term":[87,100],"arising":[88],"from":[89],"(also":[91],"function":[93],"iterate).":[97],"This":[98],"can":[101,116],"be":[102],"interpreted":[103],"perturbation":[106,115],"loss,":[109],"nature":[112],"inform":[117],"how":[118],"implicitly":[120],"(anti-)regularizes":[121],"dynamics.":[124],"As":[125],"application":[127],"our":[129],"theory,":[130],"we":[131],"find":[132],"Lion":[134],"does":[135],"not":[136],"have":[137],"kind":[139],"implicit":[141],"anti-regularization":[142],"induced":[143],"AdamW":[147],"does,":[148],"providing":[149],"theory-based":[151],"explanation":[152],"Lion's":[154],"better":[155],"generalization":[156],"performance":[157],"recently":[158],"documented.":[159]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-02-06T00:00:00"}