{"id":"https://openalex.org/W2949763090","doi":"https://doi.org/10.1088/2632-2153/ab81e2","title":"Adaptive learning rate clipping stabilizes learning","display_name":"Adaptive learning rate clipping stabilizes learning","publication_year":2020,"publication_date":"2020-03-01","ids":{"openalex":"https://openalex.org/W2949763090","doi":"https://doi.org/10.1088/2632-2153/ab81e2","mag":"2949763090"},"language":"en","primary_location":{"id":"doi:10.1088/2632-2153/ab81e2","is_oa":true,"landing_page_url":"https://doi.org/10.1088/2632-2153/ab81e2","pdf_url":"https://iopscience.iop.org/article/10.1088/2632-2153/ab81e2/pdf","source":{"id":"https://openalex.org/S4210200687","display_name":"Machine Learning Science and Technology","issn_l":"2632-2153","issn":["2632-2153"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320083","host_organization_name":"IOP Publishing","host_organization_lineage":["https://openalex.org/P4310320083","https://openalex.org/P4310311669"],"host_organization_lineage_names":["IOP Publishing","Institute of Physics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning: Science and Technology","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://iopscience.iop.org/article/10.1088/2632-2153/ab81e2/pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Jeffrey M Ede","orcid":"https://orcid.org/0000-0002-9358-5364"},"institutions":[{"id":"https://openalex.org/I39555362","display_name":"University of Warwick","ror":"https://ror.org/01a77tt86","country_code":"GB","type":"education","lineage":["https://openalex.org/I39555362"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Jeffrey M Ede","raw_affiliation_strings":["Department of Physics, University of Warwick, Coventry CV4 7AL United Kingdom"],"raw_orcid":"https://orcid.org/0000-0002-9358-5364","affiliations":[{"raw_affiliation_string":"Department of Physics, University of Warwick, Coventry CV4 7AL United Kingdom","institution_ids":["https://openalex.org/I39555362"]}]},{"author_position":"last","author":{"id":null,"display_name":"Richard Beanland","orcid":null},"institutions":[{"id":"https://openalex.org/I39555362","display_name":"University of Warwick","ror":"https://ror.org/01a77tt86","country_code":"GB","type":"education","lineage":["https://openalex.org/I39555362"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Richard Beanland","raw_affiliation_strings":["Department of Physics, University of Warwick, Coventry CV4 7AL United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Physics, University of Warwick, Coventry CV4 7AL United Kingdom","institution_ids":["https://openalex.org/I39555362"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I39555362"],"apc_list":{"value":1600,"currency":"GBP","value_usd":1962},"apc_paid":{"value":1600,"currency":"GBP","value_usd":1962},"fwci":2.5889,"has_fulltext":true,"cited_by_count":22,"citation_normalized_percentile":{"value":0.90223214,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":"1","issue":"1","first_page":"015011","last_page":"015011"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10857","display_name":"Advanced Electron Microscopy Techniques and Applications","score":0.8507000207901001,"subfield":{"id":"https://openalex.org/subfields/1315","display_name":"Structural Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10857","display_name":"Advanced Electron Microscopy Techniques and Applications","score":0.8507000207901001,"subfield":{"id":"https://openalex.org/subfields/1315","display_name":"Structural Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12611","display_name":"Neural Networks and Reservoir Computing","score":0.01769999973475933,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11525","display_name":"Piezoelectric Actuators and Control","score":0.011500000022351742,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/clipping","display_name":"Clipping (morphology)","score":0.5835999846458435},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.5777000188827515},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.545199990272522},{"id":"https://openalex.org/keywords/least-mean-squares-filter","display_name":"Least mean squares filter","score":0.5177000164985657},{"id":"https://openalex.org/keywords/mean-squared-error","display_name":"Mean squared error","score":0.48240000009536743},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.42160001397132874},{"id":"https://openalex.org/keywords/hyperparameter","display_name":"Hyperparameter","score":0.3871999979019165},{"id":"https://openalex.org/keywords/complement","display_name":"Complement (music)","score":0.37959998846054077},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.3458000123500824}],"concepts":[{"id":"https://openalex.org/C2776848632","wikidata":"https://www.wikidata.org/wiki/Q853463","display_name":"Clipping (morphology)","level":2,"score":0.5835999846458435},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.5777000188827515},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.545199990272522},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5185999870300293},{"id":"https://openalex.org/C32617633","wikidata":"https://www.wikidata.org/wiki/Q1426666","display_name":"Least mean squares filter","level":3,"score":0.5177000164985657},{"id":"https://openalex.org/C139945424","wikidata":"https://www.wikidata.org/wiki/Q1940696","display_name":"Mean squared error","level":2,"score":0.48240000009536743},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.42160001397132874},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.387800008058548},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.3871999979019165},{"id":"https://openalex.org/C112313634","wikidata":"https://www.wikidata.org/wiki/Q7886648","display_name":"Complement (music)","level":5,"score":0.37959998846054077},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3677000105381012},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.3458000123500824},{"id":"https://openalex.org/C155032097","wikidata":"https://www.wikidata.org/wiki/Q798503","display_name":"Backpropagation","level":3,"score":0.3434999883174896},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.335999995470047},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3237000107765198},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.31040000915527344},{"id":"https://openalex.org/C22679943","wikidata":"https://www.wikidata.org/wiki/Q159375","display_name":"Standard deviation","level":2,"score":0.3052999973297119},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.3046000003814697},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.3034999966621399},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2985999882221222},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2973000109195709},{"id":"https://openalex.org/C92047909","wikidata":"https://www.wikidata.org/wiki/Q204034","display_name":"Hyperbolic function","level":2,"score":0.29670000076293945},{"id":"https://openalex.org/C761482","wikidata":"https://www.wikidata.org/wiki/Q118093","display_name":"Transmission (telecommunications)","level":2,"score":0.2924000024795532},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.2799000144004822},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.2773999869823456},{"id":"https://openalex.org/C186661526","wikidata":"https://www.wikidata.org/wiki/Q13647261","display_name":"Sine","level":2,"score":0.2757999897003174},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.27230000495910645},{"id":"https://openalex.org/C125014702","wikidata":"https://www.wikidata.org/wiki/Q4680749","display_name":"Adaptive learning","level":2,"score":0.2597000002861023},{"id":"https://openalex.org/C2779127903","wikidata":"https://www.wikidata.org/wiki/Q6510194","display_name":"Learning rule","level":3,"score":0.2547999918460846},{"id":"https://openalex.org/C2781002164","wikidata":"https://www.wikidata.org/wiki/Q6822311","display_name":"Meta learning (computer science)","level":3,"score":0.25119999051094055},{"id":"https://openalex.org/C150217764","wikidata":"https://www.wikidata.org/wiki/Q6803607","display_name":"Mean absolute percentage error","level":3,"score":0.25049999356269836}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1088/2632-2153/ab81e2","is_oa":true,"landing_page_url":"https://doi.org/10.1088/2632-2153/ab81e2","pdf_url":"https://iopscience.iop.org/article/10.1088/2632-2153/ab81e2/pdf","source":{"id":"https://openalex.org/S4210200687","display_name":"Machine Learning Science and Technology","issn_l":"2632-2153","issn":["2632-2153"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320083","host_organization_name":"IOP Publishing","host_organization_lineage":["https://openalex.org/P4310320083","https://openalex.org/P4310311669"],"host_organization_lineage_names":["IOP Publishing","Institute of Physics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning: Science and Technology","raw_type":"journal-article"},{"id":"pmh:oai:wrap.warwick.ac.uk:138572","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306400665","display_name":"Warwick Research Archive Portal (University of Warwick)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I39555362","host_organization_name":"University of Warwick","host_organization_lineage":["https://openalex.org/I39555362"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Journal Article"},{"id":"pmh:oai:arXiv.org:1906.09060","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1906.09060","pdf_url":"https://arxiv.org/pdf/1906.09060","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1088/2632-2153/ab81e2","is_oa":true,"landing_page_url":"https://doi.org/10.1088/2632-2153/ab81e2","pdf_url":"https://iopscience.iop.org/article/10.1088/2632-2153/ab81e2/pdf","source":{"id":"https://openalex.org/S4210200687","display_name":"Machine Learning Science and Technology","issn_l":"2632-2153","issn":["2632-2153"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320083","host_organization_name":"IOP Publishing","host_organization_lineage":["https://openalex.org/P4310320083","https://openalex.org/P4310311669"],"host_organization_lineage_names":["IOP Publishing","Institute of Physics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning: Science and Technology","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1431922895","display_name":"ADEPT - Advanced Devices by ElectroPlaTing","funder_award_id":"EP/N035437/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G1457043649","display_name":null,"funder_award_id":"1917382","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G3983872209","display_name":"Application of novel computing and data analysis methods in electron microscopy","funder_award_id":"1917382","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G8547666004","display_name":null,"funder_award_id":"EP/N035437/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2949763090.pdf","grobid_xml":"https://content.openalex.org/works/W2949763090.grobid-xml"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W1815076433","https://openalex.org/W2046033161","https://openalex.org/W2145339207","https://openalex.org/W2762685704","https://openalex.org/W2892655153","https://openalex.org/W2903539381","https://openalex.org/W2963039693","https://openalex.org/W3027859052","https://openalex.org/W6631190155","https://openalex.org/W6631943919","https://openalex.org/W6637242042","https://openalex.org/W6638667902","https://openalex.org/W6674914833","https://openalex.org/W6684191040","https://openalex.org/W6686164453","https://openalex.org/W6687483927","https://openalex.org/W6695676441","https://openalex.org/W6713134421","https://openalex.org/W6727249380","https://openalex.org/W6739696289","https://openalex.org/W6743446608","https://openalex.org/W6749538638","https://openalex.org/W6756137178","https://openalex.org/W6770398692","https://openalex.org/W6996569244"],"related_works":[],"abstract_inverted_index":{"Abstract":[0],"Artificial":[1],"neural":[2],"network":[3],"training":[4,22,110,116],"with":[5,14,23,97],"gradient":[6,94],"descent":[7],"can":[8,74],"be":[9,75],"destabilized":[10],"by":[11],"\u2018bad":[12],"batches\u2019":[13],"high":[15,27,33],"losses.":[16],"This":[17],"is":[18,62,71,84,117,142],"often":[19],"problematic":[20],"for":[21,105,129],"small":[24],"batch":[25,82],"sizes,":[26],"order":[28],"loss":[29,79],"functions":[30],"or":[31,81],"unstably":[32],"learning":[34,43,67],"rates.":[35],"To":[36],"stabilize":[37],"learning,":[38],"we":[39],"have":[40],"developed":[41],"adaptive":[42],"rate":[44],"clipping":[45],"(ALRC)":[46],"to":[47,51,64,77,86],"limit":[48],"backpropagated":[49,93],"losses":[50],"a":[52],"number":[53],"of":[54],"standard":[55],"deviations":[56],"above":[57],"their":[58],"running":[59],"means.":[60],"ALRC":[61,123],"designed":[63],"complement":[65],"existing":[66],"algorithms:":[68],"Our":[69,139],"algorithm":[70],"computationally":[72],"inexpensive,":[73],"applied":[76],"any":[78],"function":[80],"size,":[83],"robust":[85],"hyperparameter":[87],"choices":[88],"and":[89,135],"does":[90],"not":[91],"affect":[92],"distributions.":[95],"Experiments":[96],"CIFAR-10":[98],"supersampling":[99,134],"show":[100,121],"that":[101,122],"ALCR":[102],"decreases":[103,124],"errors":[104,128],"unstable":[106,125],"mean":[107,113,126],"quartic":[108],"error":[109,115],"while":[111],"stable":[112],"squared":[114,127],"unaffected.":[118],"We":[119],"also":[120],"scanning":[130],"transmission":[131],"electron":[132],"microscopy":[133],"partial":[136],"scan":[137],"completion.":[138],"source":[140],"code":[141],"available":[143],"at":[144],"https://github.com/Jeffrey-Ede/ALRC":[145],".":[146]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":6}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2019-06-27T00:00:00"}