{"id":"https://openalex.org/W4380520271","doi":"https://doi.org/10.1109/tnnls.2023.3279381","title":"A Unified Analysis of AdaGrad With Weighted Aggregation and Momentum Acceleration","display_name":"A Unified Analysis of AdaGrad With Weighted Aggregation and Momentum Acceleration","publication_year":2023,"publication_date":"2023-06-13","ids":{"openalex":"https://openalex.org/W4380520271","doi":"https://doi.org/10.1109/tnnls.2023.3279381","pmid":"https://pubmed.ncbi.nlm.nih.gov/37310828"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2023.3279381","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2023.3279381","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042802004","display_name":"Li Shen","orcid":"https://orcid.org/0000-0001-5659-3464"},"institutions":[{"id":"https://openalex.org/I4210103986","display_name":"Jingdong (China)","ror":"https://ror.org/01dkjkq64","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210103986"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li Shen","raw_affiliation_strings":["JD Explore Academy, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-5659-3464","affiliations":[{"raw_affiliation_string":"JD Explore Academy, Beijing, China","institution_ids":["https://openalex.org/I4210103986"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027220244","display_name":"Congliang Chen","orcid":"https://orcid.org/0000-0002-9795-4200"},"institutions":[{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Congliang Chen","raw_affiliation_strings":["School of Science and Engineering, The Chinese University of Hong Kong (Shenzhen), Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-9795-4200","affiliations":[{"raw_affiliation_string":"School of Science and Engineering, The Chinese University of Hong Kong (Shenzhen), Shenzhen, China","institution_ids":["https://openalex.org/I4210116924"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026617941","display_name":"Fangyu Zou","orcid":"https://orcid.org/0000-0002-9017-122X"},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fangyu Zou","raw_affiliation_strings":["Meta Platforms Inc., Menlo Park, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-9017-122X","affiliations":[{"raw_affiliation_string":"Meta Platforms Inc., Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075329194","display_name":"Zequn Jie","orcid":"https://orcid.org/0000-0002-3038-5891"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zequn Jie","raw_affiliation_strings":["Meituan Inc., Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-3038-5891","affiliations":[{"raw_affiliation_string":"Meituan Inc., Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011253128","display_name":"Ju Sun","orcid":"https://orcid.org/0000-0002-2017-5903"},"institutions":[{"id":"https://openalex.org/I130238516","display_name":"University of Minnesota","ror":"https://ror.org/017zqws13","country_code":"US","type":"education","lineage":["https://openalex.org/I130238516"]},{"id":"https://openalex.org/I4210101327","display_name":"Twin Cities Orthopedics","ror":"https://ror.org/01en4s460","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210101327"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ju Sun","raw_affiliation_strings":["University of Minnesota at Twin Cities, Minneapolis, MN, USA"],"raw_orcid":"https://orcid.org/0000-0002-2017-5903","affiliations":[{"raw_affiliation_string":"University of Minnesota at Twin Cities, Minneapolis, MN, USA","institution_ids":["https://openalex.org/I4210101327","https://openalex.org/I130238516"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100431792","display_name":"Wei Liu","orcid":"https://orcid.org/0000-0002-3865-8145"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Liu","raw_affiliation_strings":["Tencent AI Lab, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-3865-8145","affiliations":[{"raw_affiliation_string":"Tencent AI Lab, Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.5766,"has_fulltext":false,"cited_by_count":27,"citation_normalized_percentile":{"value":0.94187997,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"35","issue":"10","first_page":"14482","last_page":"14490"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10500","display_name":"Sparse and Compressive Sensing Techniques","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/momentum","display_name":"Momentum (technical analysis)","score":0.6715872883796692},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.6068898439407349},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.5285671353340149},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.4734511077404022},{"id":"https://openalex.org/keywords/applied-mathematics","display_name":"Applied mathematics","score":0.4553629755973816},{"id":"https://openalex.org/keywords/acceleration","display_name":"Acceleration","score":0.44495147466659546},{"id":"https://openalex.org/keywords/ball","display_name":"Ball (mathematics)","score":0.42576029896736145},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.40561267733573914},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3595791459083557},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.35735154151916504},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3476465344429016},{"id":"https://openalex.org/keywords/mathematical-analysis","display_name":"Mathematical analysis","score":0.2943189740180969},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.1827460527420044},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.17640694975852966},{"id":"https://openalex.org/keywords/classical-mechanics","display_name":"Classical mechanics","score":0.129315584897995}],"concepts":[{"id":"https://openalex.org/C60718061","wikidata":"https://www.wikidata.org/wiki/Q1414747","display_name":"Momentum (technical analysis)","level":2,"score":0.6715872883796692},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.6068898439407349},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5285671353340149},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.4734511077404022},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.4553629755973816},{"id":"https://openalex.org/C117896860","wikidata":"https://www.wikidata.org/wiki/Q11376","display_name":"Acceleration","level":2,"score":0.44495147466659546},{"id":"https://openalex.org/C122041747","wikidata":"https://www.wikidata.org/wiki/Q838611","display_name":"Ball (mathematics)","level":2,"score":0.42576029896736145},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40561267733573914},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3595791459083557},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.35735154151916504},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3476465344429016},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.2943189740180969},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.1827460527420044},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.17640694975852966},{"id":"https://openalex.org/C74650414","wikidata":"https://www.wikidata.org/wiki/Q11397","display_name":"Classical mechanics","level":1,"score":0.129315584897995},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2023.3279381","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2023.3279381","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:37310828","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37310828","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W1987083649","https://openalex.org/W1988720110","https://openalex.org/W2097117768","https://openalex.org/W2108598243","https://openalex.org/W2112796928","https://openalex.org/W2124541940","https://openalex.org/W2137731592","https://openalex.org/W2146502635","https://openalex.org/W2194775991","https://openalex.org/W2789621689","https://openalex.org/W2808402975","https://openalex.org/W2936995161","https://openalex.org/W2963433607","https://openalex.org/W2963446712","https://openalex.org/W2963470657","https://openalex.org/W2963698657","https://openalex.org/W2968917279","https://openalex.org/W3091225957","https://openalex.org/W3118608800","https://openalex.org/W3120855041","https://openalex.org/W3203803206","https://openalex.org/W4200631749","https://openalex.org/W4237284205","https://openalex.org/W4382318707","https://openalex.org/W4385573252","https://openalex.org/W4388116480","https://openalex.org/W6604254268","https://openalex.org/W6630944920","https://openalex.org/W6631190155","https://openalex.org/W6676105031","https://openalex.org/W6680196509","https://openalex.org/W6681435938","https://openalex.org/W6684191040","https://openalex.org/W6734509502","https://openalex.org/W6738336094","https://openalex.org/W6740256410","https://openalex.org/W6747620207","https://openalex.org/W6748896701","https://openalex.org/W6751189704","https://openalex.org/W6752775508","https://openalex.org/W6753892653","https://openalex.org/W6754654111","https://openalex.org/W6758855485","https://openalex.org/W6761030284","https://openalex.org/W6762626082","https://openalex.org/W6762990414","https://openalex.org/W6767152389","https://openalex.org/W6767164110","https://openalex.org/W6767501881","https://openalex.org/W6787972765","https://openalex.org/W6788926732","https://openalex.org/W6805973258","https://openalex.org/W6847143135","https://openalex.org/W6850012631"],"related_works":["https://openalex.org/W2565094479","https://openalex.org/W2390829436","https://openalex.org/W1989791859","https://openalex.org/W602859758","https://openalex.org/W1971289376","https://openalex.org/W2379101322","https://openalex.org/W3007093918","https://openalex.org/W4308706819","https://openalex.org/W3046141244","https://openalex.org/W4285483873"],"abstract_inverted_index":{"Integrating":[0],"adaptive":[1,20,106,146],"learning":[2,107,113,147,188],"rate":[3,108,135],"and":[4,29,68,92,99,119,151,169,183,190],"momentum":[5,67,82,91,98],"techniques":[6],"into":[7],"stochastic":[8,21,55,139],"gradient":[9,96],"descent":[10],"(SGD)":[11],"leads":[12],"to":[13,154],"a":[14,42,80,103,163],"large":[15,43],"class":[16],"of":[17,34,48,115,149,174],"efficiently":[18],"accelerated":[19,95],"algorithms,":[22],"such":[23],"as":[24],"AdaGrad,":[25,116,180],"RMSProp,":[26],"Adam,":[27,118,182],"AccAdaGrad,":[28,117],"so":[30],"on.":[31],"In":[32],"spite":[33],"their":[35,46],"effectiveness":[36],"in":[37,45,51,128,136,159],"practice,":[38],"there":[39],"is":[40],"still":[41],"gap":[44],"theories":[47],"convergences,":[49],"especially":[50],"the":[52,73,87,93,112,137,145],"difficult":[53],"nonconvex":[54,138],"setting.":[56,140],"To":[57],"fill":[58],"this":[59],"gap,":[60],"we":[61,123,130],"propose":[62],"weighted":[63,105],"AdaGrad":[64],"with":[65,178],"unified":[66,81],"dubbed":[69],"AdaUSM,":[70,129,160],"which":[71],"has":[72],"main":[74],"characteristics":[75],"that:":[76],"1)":[77],"it":[78,101],"incorporates":[79],"scheme":[83],"that":[84,109,144],"covers":[85],"both":[86],"heavy":[88],"ball":[89],"(HB)":[90],"Nesterov":[94],"(NAG)":[97],"2)":[100],"adopts":[102],"novel":[104],"can":[110],"unify":[111],"rates":[114,148],"RMSProp.":[120,170],"Moreover,":[121],"when":[122],"take":[124],"polynomially":[125],"growing":[126,157],"weights":[127,158],"obtain":[131],"its":[132],"O(log(T)/\u221aT)":[133],"convergence":[134],"We":[141],"also":[142,193],"show":[143],"Adam":[150,168],"RMSProp":[152],"correspond":[153],"taking":[155],"exponentially":[156],"thereby":[161],"providing":[162],"new":[164],"perspective":[165],"for":[166],"understanding":[167],"Finally,":[171],"comparative":[172],"experiments":[173],"AdaUSM":[175],"against":[176],"SGD":[177],"momentum,":[179],"AdaEMA,":[181],"AMSGrad":[184],"on":[185],"various":[186],"deep":[187],"models":[189],"datasets":[191],"are":[192],"carried":[194],"out.":[195]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":3}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}