{"id":"https://openalex.org/W3203168519","doi":"https://doi.org/10.1137/20m1353010","title":"Plateau Phenomenon in Gradient Descent Training of RELU Networks: Explanation, Quantification, and Avoidance","display_name":"Plateau Phenomenon in Gradient Descent Training of RELU Networks: Explanation, Quantification, and Avoidance","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3203168519","doi":"https://doi.org/10.1137/20m1353010","mag":"3203168519"},"language":"en","primary_location":{"id":"doi:10.1137/20m1353010","is_oa":false,"landing_page_url":"https://doi.org/10.1137/20m1353010","pdf_url":null,"source":{"id":"https://openalex.org/S165512578","display_name":"SIAM Journal on Scientific Computing","issn_l":"1064-8275","issn":["1064-8275","1095-7197"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320508","host_organization_name":"Society for Industrial and Applied Mathematics","host_organization_lineage":["https://openalex.org/P4310320508"],"host_organization_lineage_names":["Society for Industrial and Applied Mathematics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SIAM Journal on Scientific Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.osti.gov/servlets/purl/1786969","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091196582","display_name":"Mark Ainsworth","orcid":"https://orcid.org/0000-0002-4899-1048"},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mark Ainsworth","raw_affiliation_strings":["Brown Univ., Providence, RI (United States)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brown Univ., Providence, RI (United States)","institution_ids":["https://openalex.org/I27804330"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023632213","display_name":"Yeonjong Shin","orcid":"https://orcid.org/0000-0003-4577-1979"},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yeonjong Shin","raw_affiliation_strings":["Brown Univ., Providence, RI (United States)"],"raw_orcid":"https://orcid.org/0000-0003-4577-1979","affiliations":[{"raw_affiliation_string":"Brown Univ., Providence, RI (United States)","institution_ids":["https://openalex.org/I27804330"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I27804330"],"apc_list":null,"apc_paid":null,"fwci":1.2999,"has_fulltext":true,"cited_by_count":28,"citation_normalized_percentile":{"value":0.83990013,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":"43","issue":"5","first_page":"A3438","last_page":"A3468"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10558","display_name":"Advancements in Semiconductor Devices and Circuit Design","score":0.9850999712944031,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11975","display_name":"Evolutionary Algorithms and Applications","score":0.9771999716758728,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.6538223028182983},{"id":"https://openalex.org/keywords/plateau","display_name":"Plateau (mathematics)","score":0.5591630935668945},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5455155372619629},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.5299191474914551},{"id":"https://openalex.org/keywords/descent","display_name":"Descent (aeronautics)","score":0.5270096063613892},{"id":"https://openalex.org/keywords/phenomenon","display_name":"Phenomenon","score":0.47642025351524353},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.40293943881988525},{"id":"https://openalex.org/keywords/applied-mathematics","display_name":"Applied mathematics","score":0.38919419050216675},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.38768965005874634},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3670389652252197},{"id":"https://openalex.org/keywords/mathematical-analysis","display_name":"Mathematical analysis","score":0.26404446363449097},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.2566683888435364},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.09987565875053406}],"concepts":[{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.6538223028182983},{"id":"https://openalex.org/C2780030769","wikidata":"https://www.wikidata.org/wiki/Q4968575","display_name":"Plateau (mathematics)","level":2,"score":0.5591630935668945},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5455155372619629},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.5299191474914551},{"id":"https://openalex.org/C2776637919","wikidata":"https://www.wikidata.org/wiki/Q624380","display_name":"Descent (aeronautics)","level":2,"score":0.5270096063613892},{"id":"https://openalex.org/C50335755","wikidata":"https://www.wikidata.org/wiki/Q483247","display_name":"Phenomenon","level":2,"score":0.47642025351524353},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.40293943881988525},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.38919419050216675},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.38768965005874634},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3670389652252197},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.26404446363449097},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.2566683888435364},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.09987565875053406},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1137/20m1353010","is_oa":false,"landing_page_url":"https://doi.org/10.1137/20m1353010","pdf_url":null,"source":{"id":"https://openalex.org/S165512578","display_name":"SIAM Journal on Scientific Computing","issn_l":"1064-8275","issn":["1064-8275","1095-7197"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320508","host_organization_name":"Society for Industrial and Applied Mathematics","host_organization_lineage":["https://openalex.org/P4310320508"],"host_organization_lineage_names":["Society for Industrial and Applied Mathematics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SIAM Journal on Scientific Computing","raw_type":"journal-article"},{"id":"pmh:oai:osti.gov:1786969","is_oa":true,"landing_page_url":"https://www.osti.gov/biblio/1786969","pdf_url":"https://www.osti.gov/servlets/purl/1786969","source":{"id":"https://openalex.org/S4306402487","display_name":"OSTI OAI (U.S. Department of Energy Office of Scientific and Technical Information)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I139351228","host_organization_name":"Office of Scientific and Technical Information","host_organization_lineage":["https://openalex.org/I139351228"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:osti.gov:1786969","is_oa":true,"landing_page_url":"https://www.osti.gov/biblio/1786969","pdf_url":"https://www.osti.gov/servlets/purl/1786969","source":{"id":"https://openalex.org/S4306402487","display_name":"OSTI OAI (U.S. Department of Energy Office of Scientific and Technical Information)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I139351228","host_organization_name":"Office of Scientific and Technical Information","host_organization_lineage":["https://openalex.org/I139351228"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3203168519.pdf"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W1988485873","https://openalex.org/W1995842804","https://openalex.org/W1999590492","https://openalex.org/W2020107577","https://openalex.org/W2024690006","https://openalex.org/W2067596735","https://openalex.org/W2079224763","https://openalex.org/W2103496339","https://openalex.org/W2125911849","https://openalex.org/W2158581396","https://openalex.org/W2166116275","https://openalex.org/W2528305538","https://openalex.org/W2895143189","https://openalex.org/W2919115771","https://openalex.org/W2963095610","https://openalex.org/W2963146412","https://openalex.org/W2963791871","https://openalex.org/W2990138404","https://openalex.org/W4247924304"],"related_works":["https://openalex.org/W2486267010","https://openalex.org/W4300450609","https://openalex.org/W4386931570","https://openalex.org/W2391010541","https://openalex.org/W2357367123","https://openalex.org/W4388930439","https://openalex.org/W2387276901","https://openalex.org/W2385953334","https://openalex.org/W2351303360","https://openalex.org/W2015288657"],"abstract_inverted_index":{"The":[0,85,124],"ability":[1],"of":[2,15,23,71,80,95,99,135,144,155,161,197,206,208,220,228,254],"neural":[3,24,44],"networks":[4,25,45],"to":[5,27,33,90,102,106,128,158,182,194,264],"provide":[6],"\u2018best":[7],"in":[8,62,120,141,217],"class\u2019":[9],"approximation":[10],"across":[11],"a":[12,76,96,202,238,266,270],"wide":[13],"range":[14],"applications":[16],"is":[17,31,138,262],"well-documented.":[18],"Nevertheless,":[19],"the":[20,37,40,63,69,93,132,142,153,159,170,176,195,209,229,243,251,255],"powerful":[21],"expressivity":[22],"comes":[26],"naught":[28],"if":[29],"one":[30],"unable":[32],"effectively":[34],"train":[35],"(choose)":[36],"parameters":[38],"defining":[39],"network.":[41],"In":[42,57],"general,":[43],"are":[46,150,275],"trained":[47],"by":[48,250],"gradient":[49,210],"descent":[50],"type":[51],"optimization":[52],"methods,a":[53],"stochastic":[54],"variant":[55],"thereof.":[56],"practice,":[58],"such":[59],"methods":[60],"result":[61],"loss":[64,86],"function":[65],"decreases":[66],"rapidly":[67],"at":[68,258],"beginning":[70],"training":[72,162,230,241],"but":[73],"then,":[74],"after":[75],"relatively":[77],"small":[78],"number":[79,98,154,160,196],"steps,":[81],"significantly":[82],"slow":[83],"down.":[84],"may":[87],"even":[88],"appear":[89],"stagnate":[91],"over":[92,226],"period":[94],"large":[97],"epochs,":[100],"only":[101],"then":[103],"suddenly":[104],"start":[105],"decrease":[107],"fast":[108],"again":[109],"for":[110,168],"no":[111],"apparent":[112],"reason.":[113],"This":[114],"so-called":[115],"plateau":[116,136],"phenomenon":[117],"manifests":[118],"itself":[119],"many":[121],"learning":[122],"tasks.":[123],"present":[125],"work":[126],"aims":[127],"identify":[129],"and":[130,164,172],"quantify":[131],"root":[133],"causes":[134],"phenomenon.analysis":[137],"carried":[139],"out":[140],"setting":[143],"univariate":[145],"ReLU":[146],"networks.":[147],"No":[148],"assumptions":[149],"made":[151],"on":[152,233],"neurons":[156],"relative":[157],"data,":[163],"our":[165],"results":[166],"hold":[167],"both":[169],"lazy":[171],"adaptive":[173],"regimes.":[174],"Here,":[175],"main":[177],"findings":[178],"are:":[179],"plateaux":[180],"correspond":[181],"periods":[183],"during":[184],"which":[185,261],"activation":[186,191,256],"patterns":[187],"remain":[188],"constant,":[189],"where":[190],"pattern":[192,257],"refers":[193],"data":[198],"points":[199,216],"that":[200],"activate":[201],"given":[203],"neuron;":[204],"quantification":[205],"convergence":[207],"flow":[211],"dynamics;":[212],"and,":[213],"characterization":[214],"stationary":[215],"terms":[218],"solutions":[219],"local":[221],"least":[222],"squares":[223],"regression":[224],"lines":[225],"subsets":[227],"data.":[231],"Based":[232],"these":[234],"conclusions,":[235],"we":[236],"propose":[237],"new":[239],"iterative":[240],"method,":[242],"Active":[244],"Neuron":[245],"Least":[246],"Squares":[247],"(ANLS),":[248],"characterised":[249],"explicit":[252],"adjustment":[253],"each":[259],"step,":[260],"designed":[263],"enable":[265],"quick":[267],"exit":[268],"from":[269],"plateau.":[271],"Illustrative":[272],"numerical":[273],"examples":[274],"included":[276],"throughout.":[277]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":15},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}