{"id":"https://openalex.org/W4410932063","doi":"https://doi.org/10.1007/s11063-025-11776-4","title":"The Epochal Sawtooth Phenomenon: Unveiling Training Loss Oscillations in Adam and Other Optimizers","display_name":"The Epochal Sawtooth Phenomenon: Unveiling Training Loss Oscillations in Adam and Other Optimizers","publication_year":2025,"publication_date":"2025-06-01","ids":{"openalex":"https://openalex.org/W4410932063","doi":"https://doi.org/10.1007/s11063-025-11776-4"},"language":"en","primary_location":{"id":"doi:10.1007/s11063-025-11776-4","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11063-025-11776-4","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11063-025-11776-4.pdf","source":{"id":"https://openalex.org/S140962798","display_name":"Neural Processing Letters","issn_l":"1370-4621","issn":["1370-4621","1573-773X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural Processing Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s11063-025-11776-4.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101590020","display_name":"Qi Liu","orcid":"https://orcid.org/0009-0002-5706-3355"},"institutions":[{"id":"https://openalex.org/I116953780","display_name":"Tongji University","ror":"https://ror.org/03rc6as71","country_code":"CN","type":"education","lineage":["https://openalex.org/I116953780"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Qi Liu","raw_affiliation_strings":["Key Laboratory of Road and Traffic Engineering of the Ministry of Education, College of Transportation, Tongji University, 4800 Cao\u2019an Rd., 201804, Shanghai, Shanghai, People\u2019s Republic of China","Key Laboratory of Road and Traffic Engineering of the Ministry of Education, College of Transportation, Tongji University, 4800 Cao'an Rd., 201804, Shanghai, Shanghai, People's Republic of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Key Laboratory of Road and Traffic Engineering of the Ministry of Education, College of Transportation, Tongji University, 4800 Cao\u2019an Rd., 201804, Shanghai, Shanghai, People\u2019s Republic of China","institution_ids":["https://openalex.org/I116953780"]},{"raw_affiliation_string":"Key Laboratory of Road and Traffic Engineering of the Ministry of Education, College of Transportation, Tongji University, 4800 Cao'an Rd., 201804, Shanghai, Shanghai, People's Republic of China","institution_ids":["https://openalex.org/I116953780"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5114336589","display_name":"Wanjing Ma","orcid":null},"institutions":[{"id":"https://openalex.org/I116953780","display_name":"Tongji University","ror":"https://ror.org/03rc6as71","country_code":"CN","type":"education","lineage":["https://openalex.org/I116953780"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wanjing Ma","raw_affiliation_strings":["Key Laboratory of Road and Traffic Engineering of the Ministry of Education, College of Transportation, Tongji University, 4800 Cao\u2019an Rd., 201804, Shanghai, Shanghai, People\u2019s Republic of China","Key Laboratory of Road and Traffic Engineering of the Ministry of Education, College of Transportation, Tongji University, 4800 Cao'an Rd., 201804, Shanghai, Shanghai, People's Republic of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Key Laboratory of Road and Traffic Engineering of the Ministry of Education, College of Transportation, Tongji University, 4800 Cao\u2019an Rd., 201804, Shanghai, Shanghai, People\u2019s Republic of China","institution_ids":["https://openalex.org/I116953780"]},{"raw_affiliation_string":"Key Laboratory of Road and Traffic Engineering of the Ministry of Education, College of Transportation, Tongji University, 4800 Cao'an Rd., 201804, Shanghai, Shanghai, People's Republic of China","institution_ids":["https://openalex.org/I116953780"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5101590020"],"corresponding_institution_ids":["https://openalex.org/I116953780"],"apc_list":{"value":2390,"currency":"EUR","value_usd":2990},"apc_paid":{"value":2390,"currency":"EUR","value_usd":2990},"fwci":1.7012,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.83820759,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"57","issue":"3","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11238","display_name":"Simulation-Based Education in Healthcare","score":0.6855999827384949,"subfield":{"id":"https://openalex.org/subfields/2737","display_name":"Physiology"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11238","display_name":"Simulation-Based Education in Healthcare","score":0.6855999827384949,"subfield":{"id":"https://openalex.org/subfields/2737","display_name":"Physiology"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sawtooth-wave","display_name":"Sawtooth wave","score":0.9327225685119629},{"id":"https://openalex.org/keywords/phenomenon","display_name":"Phenomenon","score":0.81470787525177},{"id":"https://openalex.org/keywords/computational-intelligence","display_name":"Computational intelligence","score":0.5186740159988403},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5014910697937012},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.34902769327163696},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.3105320334434509},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.30213606357574463},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.249590665102005},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.21617859601974487},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.1203557550907135}],"concepts":[{"id":"https://openalex.org/C148837635","wikidata":"https://www.wikidata.org/wiki/Q1742397","display_name":"Sawtooth wave","level":2,"score":0.9327225685119629},{"id":"https://openalex.org/C50335755","wikidata":"https://www.wikidata.org/wiki/Q483247","display_name":"Phenomenon","level":2,"score":0.81470787525177},{"id":"https://openalex.org/C139502532","wikidata":"https://www.wikidata.org/wiki/Q1122090","display_name":"Computational intelligence","level":2,"score":0.5186740159988403},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5014910697937012},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.34902769327163696},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.3105320334434509},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30213606357574463},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.249590665102005},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.21617859601974487},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.1203557550907135},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s11063-025-11776-4","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11063-025-11776-4","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11063-025-11776-4.pdf","source":{"id":"https://openalex.org/S140962798","display_name":"Neural Processing Letters","issn_l":"1370-4621","issn":["1370-4621","1573-773X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural Processing Letters","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s11063-025-11776-4","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11063-025-11776-4","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11063-025-11776-4.pdf","source":{"id":"https://openalex.org/S140962798","display_name":"Neural Processing Letters","issn_l":"1370-4621","issn":["1370-4621","1573-773X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural Processing Letters","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4410932063.pdf","grobid_xml":"https://content.openalex.org/works/W4410932063.grobid-xml"},"referenced_works_count":9,"referenced_works":["https://openalex.org/W114517082","https://openalex.org/W569478347","https://openalex.org/W1981276685","https://openalex.org/W1988720110","https://openalex.org/W2963587345","https://openalex.org/W2964054038","https://openalex.org/W2973179757","https://openalex.org/W4393367792","https://openalex.org/W6702248584"],"related_works":["https://openalex.org/W1616098173","https://openalex.org/W172707731","https://openalex.org/W2049617177","https://openalex.org/W2030774588","https://openalex.org/W1636853822","https://openalex.org/W2106314783","https://openalex.org/W93076571","https://openalex.org/W2095966391","https://openalex.org/W2350692667","https://openalex.org/W2060861951"],"abstract_inverted_index":{"In":[0],"this":[1,65,214],"paper,":[2],"we":[3,13,61,125,190],"identify":[4,126],"and":[5,104],"analyze":[6,85],"a":[7,36,49,54,166,194],"recurring":[8],"training":[9,23],"loss":[10,40,56],"pattern,":[11],"which":[12,136],"term":[14],"the":[15,42,86,120,127,138,146,183,211],"Epochal":[16],"Sawtooth":[17],"Phenomenon":[18],"(ESP),":[19],"commonly":[20],"observed":[21],"during":[22,133],"with":[24,70,77],"adaptive":[25,114],"gradient-based":[26],"optimizers,":[27],"particularly":[28],"Adam":[29],"optimizer.":[30],"This":[31],"pattern":[32],"is":[33,67,172,222],"characterized":[34],"by":[35,48,119],"sharp":[37],"drop":[38],"in":[39,53,206],"at":[41,145,224],"beginning":[43,147],"of":[44,148,157,168,176,213],"each":[45,149],"epoch,":[46],"followed":[47],"gradual":[50],"increase,":[51],"resulting":[52],"sawtooth-shaped":[55],"curve.":[57],"Through":[58],"empirical":[59],"observations,":[60],"demonstrate":[62,200],"that":[63,110,154,201],"while":[64],"effect":[66,132],"most":[68],"pronounced":[69],"Adam,":[71],"it":[72],"persists,":[73],"although":[74],"less":[75],"severely,":[76],"other":[78],"optimizers":[79],"such":[80,94],"as":[81,95,165],"RMSProp.":[82],"We":[83,151,199],"empirically":[84],"mechanisms":[87],"underlying":[88],"ESP,":[89],"focusing":[90],"on":[91],"key":[92],"factors":[93],"Adam\u2019s":[96],"$$\\beta":[97,158],"$$":[98],"parameters,":[99],"batch":[100],"size,":[101],"data":[102,134],"shuffling,":[103,135],"sample":[105],"replacement.":[106],"Our":[107],"analysis":[108],"shows":[109],"ESP":[111,161,171,192,202],"arises":[112],"from":[113],"learning":[115],"rate":[116],"adjustments":[117],"controlled":[118],"second":[121],"moment":[122],"estimate.":[123],"Additionally,":[124],"\u201cimmediate":[128],"re-exposure":[129],"to":[130,140],"samples\u201d":[131],"causes":[137],"model":[139,179],"learn":[141],"or":[142],"memorize":[143],"more":[144],"epoch.":[150],"also":[152],"find":[153],"smaller":[155],"values":[156],"_2$$":[159],"exacerbate":[160],"but":[162],"can":[163,181,203],"act":[164],"form":[167],"regularization.":[169],"While":[170],"not":[173],"necessarily":[174],"indicative":[175],"overfitting,":[177],"higher":[178],"capacity":[180],"amplify":[182],"phenomenon.":[184],"To":[185],"further":[186],"support":[187],"our":[188,220],"analysis,":[189],"replicate":[191],"through":[193],"high-dimensional":[195],"quadratic":[196],"minimization":[197],"task.":[198],"emerge":[204],"even":[205],"simple":[207],"optimization":[208],"scenarios,":[209],"reinforcing":[210],"generality":[212],"pattern.":[215],"The":[216],"code":[217],"for":[218],"reproducing":[219],"experiments":[221],"available":[223],"https://github.com/qiliuchn/training-loss-pattern":[225],".":[226]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-06-13T06:13:01.061226","created_date":"2025-10-10T00:00:00"}