{"id":"https://openalex.org/W4400315380","doi":"https://doi.org/10.1109/tpami.2024.3423382","title":"Adan: Adaptive Nesterov Momentum Algorithm for Faster Optimizing Deep Models","display_name":"Adan: Adaptive Nesterov Momentum Algorithm for Faster Optimizing Deep Models","publication_year":2024,"publication_date":"2024-07-04","ids":{"openalex":"https://openalex.org/W4400315380","doi":"https://doi.org/10.1109/tpami.2024.3423382","pmid":"https://pubmed.ncbi.nlm.nih.gov/38963744"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2024.3423382","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2024.3423382","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048069849","display_name":"Xingyu Xie","orcid":"https://orcid.org/0000-0002-8756-5981"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xingyu Xie","raw_affiliation_strings":["State Key Lab of General AI, School of Intelligence Science and Technology, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-8756-5981","affiliations":[{"raw_affiliation_string":"State Key Lab of General AI, School of Intelligence Science and Technology, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010883708","display_name":"Pan Zhou","orcid":"https://orcid.org/0000-0003-3400-8943"},"institutions":[{"id":"https://openalex.org/I79891267","display_name":"Singapore Management University","ror":"https://ror.org/050qmg959","country_code":"SG","type":"education","lineage":["https://openalex.org/I79891267"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Pan Zhou","raw_affiliation_strings":["School of Computing and Information Systems, Singapore Management University, Singapore"],"raw_orcid":"https://orcid.org/0000-0003-3400-8943","affiliations":[{"raw_affiliation_string":"School of Computing and Information Systems, Singapore Management University, Singapore","institution_ids":["https://openalex.org/I79891267"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100319225","display_name":"Huan Li","orcid":"https://orcid.org/0000-0002-1286-6552"},"institutions":[{"id":"https://openalex.org/I205237279","display_name":"Nankai University","ror":"https://ror.org/01y1kjr75","country_code":"CN","type":"education","lineage":["https://openalex.org/I205237279"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huan Li","raw_affiliation_strings":["Institute of Robotics and Automatic Information Systems, College of Artificial Intelligence, Nankai University, Tianjin, China"],"raw_orcid":"https://orcid.org/0000-0002-1286-6552","affiliations":[{"raw_affiliation_string":"Institute of Robotics and Automatic Information Systems, College of Artificial Intelligence, Nankai University, Tianjin, China","institution_ids":["https://openalex.org/I205237279"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016399094","display_name":"Zhouchen Lin","orcid":"https://orcid.org/0000-0003-1493-7569"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhouchen Lin","raw_affiliation_strings":["State Key Lab of General AI, School of Intelligence Science and Technology, Institute for Artificial Intelligence, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1493-7569","affiliations":[{"raw_affiliation_string":"State Key Lab of General AI, School of Intelligence Science and Technology, Institute for Artificial Intelligence, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570","https://openalex.org/I4210100255"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100381753","display_name":"Shuicheng Yan","orcid":"https://orcid.org/0000-0001-8906-3777"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shuicheng Yan","raw_affiliation_strings":["Sea AI Lab, Singapore"],"raw_orcid":"https://orcid.org/0000-0001-8906-3777","affiliations":[{"raw_affiliation_string":"Sea AI Lab, Singapore","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":33.107,"has_fulltext":false,"cited_by_count":138,"citation_normalized_percentile":{"value":0.99821441,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"46","issue":"12","first_page":"9508","last_page":"9520"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.6985999941825867,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.6985999941825867,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.6804999709129333,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.6215000152587891,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6375530362129211},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5977684259414673},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.5584918260574341},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3719941973686218}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6375530362129211},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5977684259414673},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5584918260574341},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3719941973686218}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2024.3423382","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2024.3423382","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:38963744","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/38963744","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/13","score":0.49000000953674316,"display_name":"Climate action"}],"awards":[{"id":"https://openalex.org/G870971120","display_name":null,"funder_award_id":"2022ZD0160300","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320308258","display_name":"Qualcomm","ror":"https://ror.org/002zrf773"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":115,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1632114991","https://openalex.org/W1861492603","https://openalex.org/W1988720110","https://openalex.org/W1994616650","https://openalex.org/W1995562189","https://openalex.org/W2064675550","https://openalex.org/W2108598243","https://openalex.org/W2132211083","https://openalex.org/W2146502635","https://openalex.org/W2158782408","https://openalex.org/W2194775991","https://openalex.org/W2331143823","https://openalex.org/W2757910899","https://openalex.org/W2886463271","https://openalex.org/W2933138175","https://openalex.org/W2962719052","https://openalex.org/W2963150697","https://openalex.org/W2963341956","https://openalex.org/W2964110616","https://openalex.org/W2965373594","https://openalex.org/W2979826702","https://openalex.org/W2990045899","https://openalex.org/W2992308087","https://openalex.org/W2993258424","https://openalex.org/W3013056767","https://openalex.org/W3035649237","https://openalex.org/W3035682985","https://openalex.org/W3094502228","https://openalex.org/W3110015897","https://openalex.org/W3118608800","https://openalex.org/W3129217397","https://openalex.org/W3138516171","https://openalex.org/W3141595720","https://openalex.org/W3173690204","https://openalex.org/W3177813494","https://openalex.org/W3183892796","https://openalex.org/W3211290194","https://openalex.org/W4200631749","https://openalex.org/W4225823287","https://openalex.org/W4240391021","https://openalex.org/W4244393449","https://openalex.org/W4246193833","https://openalex.org/W4255900881","https://openalex.org/W4282919217","https://openalex.org/W4286914341","https://openalex.org/W4287329000","https://openalex.org/W4288325606","https://openalex.org/W4310428868","https://openalex.org/W4312391126","https://openalex.org/W4312443924","https://openalex.org/W4312453657","https://openalex.org/W4312820606","https://openalex.org/W4313156423","https://openalex.org/W4386221015","https://openalex.org/W4393241094","https://openalex.org/W4394576799","https://openalex.org/W4404573785","https://openalex.org/W6631190155","https://openalex.org/W6676105031","https://openalex.org/W6681435938","https://openalex.org/W6704571135","https://openalex.org/W6737650911","https://openalex.org/W6738373677","https://openalex.org/W6744513255","https://openalex.org/W6745136726","https://openalex.org/W6745375793","https://openalex.org/W6746152743","https://openalex.org/W6747620207","https://openalex.org/W6752571064","https://openalex.org/W6752663585","https://openalex.org/W6753170497","https://openalex.org/W6753892653","https://openalex.org/W6754654111","https://openalex.org/W6757107679","https://openalex.org/W6757751764","https://openalex.org/W6757817989","https://openalex.org/W6762287338","https://openalex.org/W6764109406","https://openalex.org/W6764322716","https://openalex.org/W6766673545","https://openalex.org/W6767164110","https://openalex.org/W6773732629","https://openalex.org/W6774133109","https://openalex.org/W6776488958","https://openalex.org/W6780017182","https://openalex.org/W6780511957","https://openalex.org/W6783600611","https://openalex.org/W6784094891","https://openalex.org/W6784291726","https://openalex.org/W6784591795","https://openalex.org/W6784955064","https://openalex.org/W6786412761","https://openalex.org/W6787488880","https://openalex.org/W6787972765","https://openalex.org/W6788135285","https://openalex.org/W6790475365","https://openalex.org/W6790545018","https://openalex.org/W6793630500","https://openalex.org/W6796289114","https://openalex.org/W6797063005","https://openalex.org/W6797790494","https://openalex.org/W6798182279","https://openalex.org/W6798240405","https://openalex.org/W6801655670","https://openalex.org/W6802205604","https://openalex.org/W6803727029","https://openalex.org/W6803890736","https://openalex.org/W6805973258","https://openalex.org/W6810423238","https://openalex.org/W6810673617","https://openalex.org/W6840343187","https://openalex.org/W6846665566","https://openalex.org/W6860710830","https://openalex.org/W6988181345"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2051487156","https://openalex.org/W2073681303","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2033914206","https://openalex.org/W2042327336"],"abstract_inverted_index":{"In":[0],"deep":[1,6,37,122],"learning,":[2],"different":[3,10],"kinds":[4],"of":[5,70,175,180,203],"networks":[7,155],"typically":[8],"need":[9],"optimizers,":[11],"which":[12,65],"have":[13],"to":[14,56,81,183,199,210],"be":[15],"chosen":[16],"after":[17],"multiple":[18],"trials,":[19],"making":[20],"the":[21,32,41,52,67,74,83,117,126,138,176],"training":[22,34,177],"process":[23],"inefficient.":[24],"To":[25],"relieve":[26],"this":[27],"issue":[28],"and":[29,86,145,148,156,167,194],"consistently":[30,136],"improve":[31],"model":[33],"speed":[35],"across":[36],"networks,":[38],"we":[39,97],"propose":[40],"ADAptive":[42],"Nesterov":[43,54,60],"momentum":[44,61],"algorithm,":[45],"Adan":[46,49,78,100,135,171],"for":[47,93,152],"short.":[48],"first":[50],"reformulates":[51],"vanilla":[53],"acceleration":[55],"develop":[57],"a":[58,200],"new":[59,150],"estimation":[62],"(NME)":[63],"method,":[64],"avoids":[66],"extra":[68],"overhead":[69],"computing":[71],"gradient":[72,91,114],"at":[73],"extrapolation":[75],"point.":[76],"Then":[77],"adopts":[79],"NME":[80],"estimate":[82],"gradient's":[84],"first-":[85],"second-order":[87],"moments":[88],"in":[89],"adaptive":[90],"algorithms":[92],"convergence":[94],"acceleration.":[95],"Besides,":[96],"prove":[98],"that":[99,134],"finds":[101],"an":[102],"<inline-formula><tex-math":[103,109],"notation=\"LaTeX\">$\\epsilon$</tex-math></inline-formula>-approximate":[104],"first-order":[105],"stationary":[106],"point":[107],"within":[108],"notation=\"LaTeX\">$\\mathcal":[110],"{O}(\\epsilon":[111],"^{-3.5})$</tex-math></inline-formula>":[112],"stochastic":[113,119],"complexity":[115],"on":[116,142,189],"non-convex":[118],"problems":[120],"(e.g.,":[121],"learning":[123],"problems),":[124],"matching":[125],"best-known":[127],"lower":[128],"bound.":[129],"Extensive":[130],"experimental":[131],"results":[132],"show":[133],"surpasses":[137],"corresponding":[139],"SoTA":[140,181],"optimizers":[141,182],"vision,":[143],"language,":[144],"RL":[146],"tasks":[147],"sets":[149],"SoTAs":[151],"many":[153],"popular":[154],"frameworks,":[157],"e.g.,":[158,206],"ResNet,":[159],"ConvNext,":[160],"ViT,":[161,190],"Swin,":[162],"MAE,":[163,192],"DETR,":[164],"GPT-2,":[165,191],"Transformer-XL,":[166],"BERT.":[168],"More":[169],"surprisingly,":[170],"can":[172],"use":[173],"half":[174],"cost":[178],"(epochs)":[179],"achieve":[184],"higher":[185],"or":[186],"comparable":[187],"performance":[188],"<i>etc</i>,":[193],"also":[195],"shows":[196],"great":[197],"tolerance":[198],"large":[201],"range":[202],"minibatch":[204],"size,":[205],"from":[207],"1":[208],"k":[209],"32":[211],"k.":[212]},"counts_by_year":[{"year":2026,"cited_by_count":37},{"year":2025,"cited_by_count":69},{"year":2024,"cited_by_count":30},{"year":2023,"cited_by_count":2}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}