{"id":"https://openalex.org/W2560283133","doi":"https://doi.org/10.1109/ijcnn.2017.7965845","title":"A robust adaptive stochastic gradient method for deep learning","display_name":"A robust adaptive stochastic gradient method for deep learning","publication_year":2017,"publication_date":"2017-05-01","ids":{"openalex":"https://openalex.org/W2560283133","doi":"https://doi.org/10.1109/ijcnn.2017.7965845","mag":"2560283133"},"language":"en","primary_location":{"id":"doi:10.1109/ijcnn.2017.7965845","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn.2017.7965845","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2017 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041145688","display_name":"\u00c7a\u011flar G\u00fcl\u00e7ehre","orcid":"https://orcid.org/0009-0003-4124-1687"},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Caglar Gulcehre","raw_affiliation_strings":["Universit\u00e9 de Montr\u00e9al"],"affiliations":[{"raw_affiliation_string":"Universit\u00e9 de Montr\u00e9al","institution_ids":["https://openalex.org/I70931966"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064354030","display_name":"Jose Sotelo","orcid":null},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jose Sotelo","raw_affiliation_strings":["Universit\u00e9 de Montr\u00e9al"],"affiliations":[{"raw_affiliation_string":"Universit\u00e9 de Montr\u00e9al","institution_ids":["https://openalex.org/I70931966"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039315344","display_name":"Marcin Moczulski","orcid":null},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Marcin Moczulski","raw_affiliation_strings":["University of Oxford"],"affiliations":[{"raw_affiliation_string":"University of Oxford","institution_ids":["https://openalex.org/I40120149"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5086198262","display_name":"Yoshua Bengio","orcid":"https://orcid.org/0000-0002-9322-3515"},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Yoshua Bengio","raw_affiliation_strings":["Universit\u00e9 de Montr\u00e9al"],"affiliations":[{"raw_affiliation_string":"Universit\u00e9 de Montr\u00e9al","institution_ids":["https://openalex.org/I70931966"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5041145688"],"corresponding_institution_ids":["https://openalex.org/I70931966"],"apc_list":null,"apc_paid":null,"fwci":2.0788,"has_fulltext":false,"cited_by_count":20,"citation_normalized_percentile":{"value":0.89814213,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"125","last_page":"132"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/variance-reduction","display_name":"Variance reduction","score":0.7162371873855591},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6523735523223877},{"id":"https://openalex.org/keywords/stochastic-approximation","display_name":"Stochastic approximation","score":0.6025887727737427},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.5965903997421265},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.5836902856826782},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.5835380554199219},{"id":"https://openalex.org/keywords/rate-of-convergence","display_name":"Rate of convergence","score":0.5513795614242554},{"id":"https://openalex.org/keywords/stochastic-optimization","display_name":"Stochastic optimization","score":0.5341675877571106},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.5296979546546936},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.48787355422973633},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.4761095941066742},{"id":"https://openalex.org/keywords/curvature","display_name":"Curvature","score":0.4756447970867157},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.4601709246635437},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.45736417174339294},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.44450733065605164},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.42236411571502686},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.41599082946777344},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.37243008613586426},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2355203926563263},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.22137796878814697},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.1429690420627594},{"id":"https://openalex.org/keywords/monte-carlo-method","display_name":"Monte Carlo method","score":0.1310608983039856},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.12185752391815186}],"concepts":[{"id":"https://openalex.org/C62644790","wikidata":"https://www.wikidata.org/wiki/Q3454689","display_name":"Variance reduction","level":3,"score":0.7162371873855591},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6523735523223877},{"id":"https://openalex.org/C55479107","wikidata":"https://www.wikidata.org/wiki/Q97663916","display_name":"Stochastic approximation","level":3,"score":0.6025887727737427},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.5965903997421265},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.5836902856826782},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5835380554199219},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.5513795614242554},{"id":"https://openalex.org/C194387892","wikidata":"https://www.wikidata.org/wiki/Q1747770","display_name":"Stochastic optimization","level":2,"score":0.5341675877571106},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.5296979546546936},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48787355422973633},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4761095941066742},{"id":"https://openalex.org/C195065555","wikidata":"https://www.wikidata.org/wiki/Q214881","display_name":"Curvature","level":2,"score":0.4756447970867157},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.4601709246635437},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.45736417174339294},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.44450733065605164},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.42236411571502686},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.41599082946777344},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37243008613586426},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2355203926563263},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.22137796878814697},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.1429690420627594},{"id":"https://openalex.org/C19499675","wikidata":"https://www.wikidata.org/wiki/Q232207","display_name":"Monte Carlo method","level":2,"score":0.1310608983039856},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.12185752391815186},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ijcnn.2017.7965845","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn.2017.7965845","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2017 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320314000","display_name":"Compute Canada","ror":"https://ror.org/03ty8yr27"},{"id":"https://openalex.org/F4320320994","display_name":"Canada Research Chairs","ror":"https://ror.org/0517h6h17"},{"id":"https://openalex.org/F4320321739","display_name":"Consejo Nacional de Ciencia y Tecnolog\u00eda","ror":"https://ror.org/059ex5q34"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W6908809","https://openalex.org/W19621276","https://openalex.org/W1522301498","https://openalex.org/W1606347560","https://openalex.org/W1810943226","https://openalex.org/W1813485996","https://openalex.org/W1899504021","https://openalex.org/W1982566368","https://openalex.org/W1994616650","https://openalex.org/W2064675550","https://openalex.org/W2107438106","https://openalex.org/W2107878631","https://openalex.org/W2113021982","https://openalex.org/W2120420045","https://openalex.org/W2130984546","https://openalex.org/W2142426721","https://openalex.org/W2145832734","https://openalex.org/W2146502635","https://openalex.org/W2152550252","https://openalex.org/W2157331557","https://openalex.org/W2164526460","https://openalex.org/W2914484425","https://openalex.org/W2964121744","https://openalex.org/W2964142744","https://openalex.org/W4255949318","https://openalex.org/W4293460823","https://openalex.org/W6600828528","https://openalex.org/W6631190155","https://openalex.org/W6636358008","https://openalex.org/W6676105031","https://openalex.org/W6681435938","https://openalex.org/W6681468881","https://openalex.org/W6681512967"],"related_works":["https://openalex.org/W2107438106","https://openalex.org/W4286899070","https://openalex.org/W4323366756","https://openalex.org/W3207830353","https://openalex.org/W4285259204","https://openalex.org/W3109737792","https://openalex.org/W3109198354","https://openalex.org/W937081240","https://openalex.org/W2760303966","https://openalex.org/W2964140730"],"abstract_inverted_index":{"Stochastic":[0],"gradient":[1,125],"algorithms":[2],"are":[3],"the":[4,17,21,31,38,41,47,65,71,77,81,87,91,107,122],"main":[5],"focus":[6],"of":[7,20,27,34,40,46,64,80,90],"large-scale":[8],"optimization":[9],"problems":[10],"and":[11,37],"led":[12],"to":[13,104,121],"important":[14],"successes":[15],"in":[16,43],"recent":[18],"advancement":[19],"deep":[22,113],"learning":[23,35,56,72],"algorithms.":[24,126],"The":[25,74],"convergence":[26],"SGD":[28],"depends":[29],"on":[30],"careful":[32],"choice":[33],"rate":[36,57],"amount":[39],"noise":[42],"stochastic":[44,61,92,124],"estimates":[45],"gradients.":[48,95],"In":[49,109],"this":[50],"paper,":[51],"we":[52,116],"propose":[53,98],"an":[54],"adaptive":[55],"algorithm,":[58],"which":[59],"utilizes":[60],"curvature":[62,79],"information":[63,75],"loss":[66,82],"function":[67,83],"for":[68],"automatically":[69],"tuning":[70],"rates.":[73],"about":[76],"element-wise":[78],"is":[84],"estimated":[85],"from":[86],"local":[88],"statistics":[89],"first":[93],"order":[94],"We":[96],"further":[97],"a":[99],"new":[100],"variance":[101],"reduction":[102],"technique":[103],"speed":[105],"up":[106],"convergence.":[108],"our":[110],"experiments":[111],"with":[112],"neural":[114],"networks,":[115],"obtained":[117],"better":[118],"performance":[119],"compared":[120],"popular":[123]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":1}],"updated_date":"2026-04-17T18:11:37.981687","created_date":"2025-10-10T00:00:00"}