{"id":"https://openalex.org/W3084521418","doi":"https://doi.org/10.1109/tnnls.2019.2955777","title":"diffGrad: An Optimization Method for Convolutional Neural Networks","display_name":"diffGrad: An Optimization Method for Convolutional Neural Networks","publication_year":2019,"publication_date":"2019-12-23","ids":{"openalex":"https://openalex.org/W3084521418","doi":"https://doi.org/10.1109/tnnls.2019.2955777","mag":"3084521418","pmid":"https://pubmed.ncbi.nlm.nih.gov/31880565"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2019.2955777","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2019.2955777","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048082929","display_name":"Shiv Ram Dubey","orcid":"https://orcid.org/0000-0002-4532-8996"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Shiv Ram Dubey","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026147621","display_name":"Soumendu Chakraborty","orcid":"https://orcid.org/0000-0002-8778-8229"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Soumendu Chakraborty","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087427076","display_name":"Swalpa Kumar Roy","orcid":"https://orcid.org/0000-0002-6580-3977"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Swalpa Kumar Roy","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070435685","display_name":"Snehasis Mukherjee","orcid":"https://orcid.org/0000-0002-2196-8980"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Snehasis Mukherjee","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071107603","display_name":"Satish Kumar Singh","orcid":"https://orcid.org/0000-0002-8536-4991"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Satish Kumar Singh","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5025093894","display_name":"B.B. Chaudhuri","orcid":"https://orcid.org/0000-0003-0297-8929"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bidyut Baran Chaudhuri","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5048082929"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":10.9321,"has_fulltext":false,"cited_by_count":276,"citation_normalized_percentile":{"value":0.98704308,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"31","issue":"11","first_page":"4500","last_page":"4511"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10331","display_name":"Video Surveillance and Tracking Methods","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10331","display_name":"Video Surveillance and Tracking Methods","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.7116689085960388},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6296149492263794},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.44560304284095764}],"concepts":[{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.7116689085960388},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6296149492263794},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44560304284095764}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2019.2955777","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2019.2955777","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:31880565","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/31880565","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309480","display_name":"Nvidia","ror":"https://ror.org/03jdj4y14"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":74,"referenced_works":["https://openalex.org/W6908809","https://openalex.org/W104184427","https://openalex.org/W114517082","https://openalex.org/W1522301498","https://openalex.org/W1536680647","https://openalex.org/W1686810756","https://openalex.org/W1689711448","https://openalex.org/W1885185971","https://openalex.org/W1906374873","https://openalex.org/W1980287119","https://openalex.org/W1985242443","https://openalex.org/W2093866254","https://openalex.org/W2097117768","https://openalex.org/W2102605133","https://openalex.org/W2117130368","https://openalex.org/W2117539524","https://openalex.org/W2145339207","https://openalex.org/W2146502635","https://openalex.org/W2148825261","https://openalex.org/W2163605009","https://openalex.org/W2168231600","https://openalex.org/W2176412452","https://openalex.org/W2194775991","https://openalex.org/W2250539671","https://openalex.org/W2412782625","https://openalex.org/W2484332996","https://openalex.org/W2557283755","https://openalex.org/W2613332842","https://openalex.org/W2613718673","https://openalex.org/W2687693326","https://openalex.org/W2768166594","https://openalex.org/W2785523195","https://openalex.org/W2803573348","https://openalex.org/W2803681939","https://openalex.org/W2889058905","https://openalex.org/W2911671827","https://openalex.org/W2914331134","https://openalex.org/W2949983948","https://openalex.org/W2953106684","https://openalex.org/W2963037989","https://openalex.org/W2963150697","https://openalex.org/W2963285578","https://openalex.org/W2963454111","https://openalex.org/W2963586744","https://openalex.org/W2963664311","https://openalex.org/W2963857746","https://openalex.org/W2963981733","https://openalex.org/W2978010459","https://openalex.org/W2978680233","https://openalex.org/W3118608800","https://openalex.org/W3147765605","https://openalex.org/W4297685247","https://openalex.org/W4301206121","https://openalex.org/W6600284362","https://openalex.org/W6604254268","https://openalex.org/W6620707391","https://openalex.org/W6631190155","https://openalex.org/W6637373629","https://openalex.org/W6681435938","https://openalex.org/W6681723013","https://openalex.org/W6681804681","https://openalex.org/W6684191040","https://openalex.org/W6684859321","https://openalex.org/W6685562342","https://openalex.org/W6739879593","https://openalex.org/W6747620207","https://openalex.org/W6748019269","https://openalex.org/W6751231343","https://openalex.org/W6751676527","https://openalex.org/W6761421612","https://openalex.org/W6764232654","https://openalex.org/W6765779288","https://openalex.org/W6787972765","https://openalex.org/W6793127723"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4293226380","https://openalex.org/W4396696052","https://openalex.org/W2382290278"],"abstract_inverted_index":{"Stochastic":[0],"gradient":[1,18,52,79,138,169,179],"descent":[2,80],"(SGD)":[3],"is":[4,40,62,125,150,185,202,254,285],"one":[5],"of":[6,13,32,50,58,99,103,114,192,228],"the":[7,11,22,29,51,96,129,132,135,142,147,188,193,218,226,233,257],"core":[8],"techniques":[9],"behind":[10],"success":[12],"deep":[14,59],"neural":[15,250],"networks.":[16],"The":[17,34,182,210,245,259,282],"provides":[19],"information":[20],"on":[21,95,128],"direction":[23],"in":[24,117,155,256],"which":[25],"a":[26,122,157,163,173],"function":[27],"has":[28],"steepest":[30],"rate":[31],"change.":[33],"main":[35],"problem":[36],"with":[37,230],"basic":[38],"SGD":[39],"to":[41,63,77,224,232],"change":[42,116],"by":[43],"equal-sized":[44],"steps":[45],"for":[46,68,152,167,177,275],"all":[47],"parameters,":[48],"irrespective":[49],"behavior.":[53],"Hence,":[54],"an":[55],"efficient":[56],"way":[57,158],"network":[60,251],"optimization":[61,145],"have":[64,74,162],"adaptive":[65,88],"step":[66,148,165,175],"sizes":[67],"each":[69,153],"parameter.":[70],"Recently,":[71],"several":[72],"attempts":[73],"been":[75],"made":[76,203,286],"improve":[78],"methods":[81,93,109],"such":[82,156,236],"as":[83,237],"AdaGrad,":[84,239],"AdaDelta,":[85,240],"RMSProp,":[86,241],"and":[87,134,172,220,243],"moment":[89],"estimation":[90],"(Adam).":[91],"These":[92],"rely":[94],"square":[97],"roots":[98],"exponential":[100],"moving":[101],"averages":[102],"squared":[104],"past":[105,137],"gradients.":[106,118],"Thus,":[107],"these":[108],"do":[110],"not":[111],"take":[112],"advantage":[113],"local":[115],"In":[119,141,197],"this":[120,198],"article,":[121,199],"novel":[123],"optimizer":[124],"proposed":[126,143],"based":[127],"difference":[130],"between":[131],"present":[133],"immediate":[136],"(i.e.,":[139],"diffGrad).":[140],"diffGrad":[144,229,263,271],"technique,":[146],"size":[149,166,176],"adjusted":[151],"parameter":[154],"that":[159,262,270],"it":[160],"should":[161],"larger":[164],"faster":[168],"changing":[170,180],"parameters":[171],"lower":[174,178],"parameters.":[181],"convergence":[183],"analysis":[184,201],"done":[186],"using":[187,278],"regret":[189],"bound":[190],"approach":[191],"online":[194],"learning":[195],"framework.":[196],"thorough":[200],"over":[204,217],"three":[205],"synthetic":[206],"complex":[207],"nonconvex":[208],"functions.":[209,281],"image":[211],"categorization":[212],"experiments":[213,260],"are":[214],"also":[215],"conducted":[216],"CIFAR10":[219],"CIFAR100":[221],"data":[222],"sets":[223],"observe":[225],"performance":[227],"respect":[231],"state-of-the-art":[234],"optimizers":[235],"SGDM,":[238],"AMSGrad,":[242],"Adam.":[244],"residual":[246],"unit":[247],"(ResNet)-based":[248],"convolutional":[249],"(CNN)":[252],"architecture":[253],"used":[255],"experiments.":[258],"show":[261,269],"outperforms":[264],"other":[265],"optimizers.":[266],"Also,":[267],"we":[268],"performs":[272],"uniformly":[273],"well":[274],"training":[276],"CNN":[277],"different":[279],"activation":[280],"source":[283],"code":[284],"publicly":[287],"available":[288],"at":[289],"https://github.com/shivram1987/diffGrad.":[290]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":42},{"year":2024,"cited_by_count":50},{"year":2023,"cited_by_count":73},{"year":2022,"cited_by_count":45},{"year":2021,"cited_by_count":47},{"year":2020,"cited_by_count":15}],"updated_date":"2026-04-12T07:58:50.170612","created_date":"2025-10-10T00:00:00"}