{"id":"https://openalex.org/W2989753407","doi":"https://doi.org/10.1109/tnnls.2022.3155181","title":"Learning Rate Dropout","display_name":"Learning Rate Dropout","publication_year":2022,"publication_date":"2022-03-14","ids":{"openalex":"https://openalex.org/W2989753407","doi":"https://doi.org/10.1109/tnnls.2022.3155181","mag":"2989753407","pmid":"https://pubmed.ncbi.nlm.nih.gov/35286266"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2022.3155181","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2022.3155181","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058405643","display_name":"Huangxing Lin","orcid":"https://orcid.org/0000-0002-2334-4987"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Huangxing Lin","raw_affiliation_strings":["School of Informatics and National Institute for Data Science in Health and Medicine, Xiamen University, Xiamen, China"],"affiliations":[{"raw_affiliation_string":"School of Informatics and National Institute for Data Science in Health and Medicine, Xiamen University, Xiamen, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014165464","display_name":"Weihong Zeng","orcid":"https://orcid.org/0000-0003-3853-3630"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weihong Zeng","raw_affiliation_strings":["School of Informatics, Xiamen University, Xiamen, China"],"affiliations":[{"raw_affiliation_string":"School of Informatics, Xiamen University, Xiamen, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052754372","display_name":"Yihong Zhuang","orcid":"https://orcid.org/0000-0002-3176-3217"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yihong Zhuang","raw_affiliation_strings":["School of Informatics, Xiamen University, Xiamen, China"],"affiliations":[{"raw_affiliation_string":"School of Informatics, Xiamen University, Xiamen, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052820597","display_name":"Xinghao Ding","orcid":"https://orcid.org/0000-0003-2288-5287"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xinghao Ding","raw_affiliation_strings":["School of Informatics, National Institute for Data Science in Health and Medicine, and Institute of Artificial Intelligence, Xiamen University, Xiamen, China"],"affiliations":[{"raw_affiliation_string":"School of Informatics, National Institute for Data Science in Health and Medicine, and Institute of Artificial Intelligence, Xiamen University, Xiamen, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100772804","display_name":"Yue Huang","orcid":"https://orcid.org/0000-0002-3913-9400"},"institutions":[{"id":"https://openalex.org/I191208505","display_name":"Xiamen University","ror":"https://ror.org/00mcjh785","country_code":"CN","type":"education","lineage":["https://openalex.org/I191208505"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yue Huang","raw_affiliation_strings":["School of Informatics and Institute of Artificial Intelligence, Xiamen University, Xiamen, China"],"affiliations":[{"raw_affiliation_string":"School of Informatics and Institute of Artificial Intelligence, Xiamen University, Xiamen, China","institution_ids":["https://openalex.org/I191208505"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5110662445","display_name":"John Paisley","orcid":null},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John Paisley","raw_affiliation_strings":["Department of Electrical Engineering, Data Science Institute, Columbia University, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering, Data Science Institute, Columbia University, New York, NY, USA","institution_ids":["https://openalex.org/I78577930"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5058405643"],"corresponding_institution_ids":["https://openalex.org/I191208505"],"apc_list":null,"apc_paid":null,"fwci":1.1073,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.77556421,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"34","issue":"11","first_page":"9029","last_page":"9039"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dropout","display_name":"Dropout (neural networks)","score":0.7821516990661621},{"id":"https://openalex.org/keywords/overfitting","display_name":"Overfitting","score":0.6820420622825623},{"id":"https://openalex.org/keywords/local-optimum","display_name":"Local optimum","score":0.6377244591712952},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6119043231010437},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.558927059173584},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.5372405648231506},{"id":"https://openalex.org/keywords/rate-of-convergence","display_name":"Rate of convergence","score":0.5359182357788086},{"id":"https://openalex.org/keywords/saddle-point","display_name":"Saddle point","score":0.5153093338012695},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.47250258922576904},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.4552832841873169},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.44499412178993225},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.43369239568710327},{"id":"https://openalex.org/keywords/momentum","display_name":"Momentum (technical analysis)","score":0.43003562092781067},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3989604115486145},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.2976200580596924},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2702977657318115}],"concepts":[{"id":"https://openalex.org/C2776145597","wikidata":"https://www.wikidata.org/wiki/Q25339462","display_name":"Dropout (neural networks)","level":2,"score":0.7821516990661621},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.6820420622825623},{"id":"https://openalex.org/C141934464","wikidata":"https://www.wikidata.org/wiki/Q3305386","display_name":"Local optimum","level":2,"score":0.6377244591712952},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6119043231010437},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.558927059173584},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.5372405648231506},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.5359182357788086},{"id":"https://openalex.org/C2681867","wikidata":"https://www.wikidata.org/wiki/Q690935","display_name":"Saddle point","level":2,"score":0.5153093338012695},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.47250258922576904},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.4552832841873169},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.44499412178993225},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.43369239568710327},{"id":"https://openalex.org/C60718061","wikidata":"https://www.wikidata.org/wiki/Q1414747","display_name":"Momentum (technical analysis)","level":2,"score":0.43003562092781067},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3989604115486145},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2976200580596924},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2702977657318115},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2022.3155181","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2022.3155181","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:35286266","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/35286266","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G383496509","display_name":null,"funder_award_id":"2019HZ020009","funder_id":"https://openalex.org/F4320317314","funder_display_name":"Science and Technology Projects of Fujian Province"},{"id":"https://openalex.org/G5115423372","display_name":null,"funder_award_id":"61971369","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5339618318","display_name":null,"funder_award_id":"82172033","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6130862679","display_name":null,"funder_award_id":"U19B2031","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7418738665","display_name":null,"funder_award_id":"52105126","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320317314","display_name":"Science and Technology Projects of Fujian Province","ror":null},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":69,"referenced_works":["https://openalex.org/W4919037","https://openalex.org/W6908809","https://openalex.org/W35527955","https://openalex.org/W1522301498","https://openalex.org/W1836465849","https://openalex.org/W1903029394","https://openalex.org/W1923697677","https://openalex.org/W1980287119","https://openalex.org/W1994616650","https://openalex.org/W2000769684","https://openalex.org/W2002745466","https://openalex.org/W2024060531","https://openalex.org/W2031489346","https://openalex.org/W2095705004","https://openalex.org/W2095984592","https://openalex.org/W2108677974","https://openalex.org/W2111406701","https://openalex.org/W2112796928","https://openalex.org/W2117539524","https://openalex.org/W2124136621","https://openalex.org/W2144794286","https://openalex.org/W2145094598","https://openalex.org/W2148825261","https://openalex.org/W2157444450","https://openalex.org/W2163605009","https://openalex.org/W2194775991","https://openalex.org/W2263490141","https://openalex.org/W2345474290","https://openalex.org/W2504108613","https://openalex.org/W2549139847","https://openalex.org/W2560023338","https://openalex.org/W2618530766","https://openalex.org/W2619516334","https://openalex.org/W2766736793","https://openalex.org/W2785523195","https://openalex.org/W2798391154","https://openalex.org/W2893810361","https://openalex.org/W2907225497","https://openalex.org/W2949935872","https://openalex.org/W2951266961","https://openalex.org/W2963208657","https://openalex.org/W2963227127","https://openalex.org/W2963446712","https://openalex.org/W2963563573","https://openalex.org/W2963703360","https://openalex.org/W2964309882","https://openalex.org/W2968917279","https://openalex.org/W3034995656","https://openalex.org/W3106250896","https://openalex.org/W3110386488","https://openalex.org/W3118608800","https://openalex.org/W4297781872","https://openalex.org/W6600213771","https://openalex.org/W6600284362","https://openalex.org/W6601425427","https://openalex.org/W6631190155","https://openalex.org/W6638667902","https://openalex.org/W6674330103","https://openalex.org/W6681723013","https://openalex.org/W6684191040","https://openalex.org/W6692956712","https://openalex.org/W6697168296","https://openalex.org/W6738491991","https://openalex.org/W6747620207","https://openalex.org/W6757751764","https://openalex.org/W6764214684","https://openalex.org/W6767164110","https://openalex.org/W6785652829","https://openalex.org/W6787972765"],"related_works":["https://openalex.org/W4298017035","https://openalex.org/W3128220493","https://openalex.org/W2792147139","https://openalex.org/W4226354336","https://openalex.org/W3110700750","https://openalex.org/W2998675825","https://openalex.org/W4394636190","https://openalex.org/W2736804899","https://openalex.org/W2897443685","https://openalex.org/W4307654087"],"abstract_inverted_index":{"Optimization":[0],"algorithms":[1,19],"are":[2,82,104,109],"of":[3,89],"great":[4],"importance":[5],"to":[6,30,45,56,91,132],"efficiently":[7],"and":[8,49,138],"effectively":[9],"train":[10],"a":[11,40],"deep":[12],"neural":[13],"network.":[14],"However,":[15],"the":[16,54,60,87,97,114,117,128,135],"existing":[17],"optimization":[18],"show":[20,143],"unsatisfactory":[21],"convergence":[22,48,98],"behavior,":[23],"either":[24],"slowly":[25],"converging":[26],"or":[27],"not":[28,80,105],"seeking":[29],"avoid":[31],"bad":[32,139],"local":[33,140],"optima.":[34,141],"Learning":[35],"rate":[36,78],"dropout":[37],"(LRD)":[38],"is":[39,79,146],"new":[41],"gradient":[42],"descent":[43],"technique":[44],"motivate":[46],"faster":[47],"better":[50],"generalization.":[51],"LRD":[52,85,145],"aids":[53],"optimizer":[55,115,129],"actively":[57],"explore":[58],"in":[59,149],"parameter":[61,125],"space":[62],"by":[63,113],"randomly":[64],"dropping":[65],"some":[66],"learning":[67,77],"rates":[68],"(to":[69],"0);":[70],"at":[71,123],"each":[72,95],"iteration,":[73,96],"only":[74],"parameters":[75,90,102],"whose":[76],"0":[81],"updated.":[83],"Since":[84],"reduces":[86],"number":[88],"be":[92],"updated":[93],"for":[94,116],"becomes":[99],"easier.":[100],"For":[101],"that":[103,144],"updated,":[106],"their":[107],"gradients":[108,122],"accumulated":[110],"(e.g.,":[111],"momentum)":[112],"next":[118],"update.":[119],"Accumulating":[120],"multiple":[121],"fixed":[124],"positions":[126],"gives":[127],"more":[130],"energy":[131],"escape":[133],"from":[134],"saddle":[136],"point":[137],"Experiments":[142],"surprisingly":[147],"effective":[148],"accelerating":[150],"training":[151],"while":[152],"preventing":[153],"overfitting.":[154]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}