{"id":"https://openalex.org/W3021189130","doi":"https://doi.org/10.1109/jsait.2020.2991332","title":"Toward Moderate Overparameterization: Global Convergence Guarantees for Training Shallow Neural Networks","display_name":"Toward Moderate Overparameterization: Global Convergence Guarantees for Training Shallow Neural Networks","publication_year":2020,"publication_date":"2020-04-29","ids":{"openalex":"https://openalex.org/W3021189130","doi":"https://doi.org/10.1109/jsait.2020.2991332","mag":"3021189130"},"language":"en","primary_location":{"id":"doi:10.1109/jsait.2020.2991332","doi":"https://doi.org/10.1109/jsait.2020.2991332","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jsait.2020.2991332","pdf_url":null,"source":{"id":"https://openalex.org/S4210211895","display_name":"IEEE Journal on Selected Areas in Information Theory","issn_l":"2641-8770","issn":["2641-8770"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-b","jufo-1","norway-1"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal on Selected Areas in Information Theory","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050547472","display_name":"Samet Oymak","orcid":"https://orcid.org/0000-0001-5203-0752","observed_orcids":["https://orcid.org/0000-0001-5203-0752"]},"institutions":[{"id":"https://openalex.org/I103635307","display_name":"University of California, Riverside","ror":"https://ror.org/03nawhv43","country_code":"US","type":"education","lineage":["https://openalex.org/I103635307"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Samet Oymak","raw_affiliation_strings":["University of California at Riverside, Riverside, USA"],"raw_orcid":"https://orcid.org/0000-0001-5203-0752","affiliations":[{"raw_affiliation_string":"University of California at Riverside, Riverside, USA","institution_ids":["https://openalex.org/I103635307"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046962187","display_name":"Mahdi Soltanolkotabi","orcid":"https://orcid.org/0000-0003-2101-6418","observed_orcids":["https://orcid.org/0000-0003-2101-6418"]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mahdi Soltanolkotabi","raw_affiliation_strings":["University of Southern California, Los Angeles, USA"],"raw_orcid":"https://orcid.org/0000-0003-2101-6418","affiliations":[{"raw_affiliation_string":"University of Southern California, Los Angeles, USA","institution_ids":["https://openalex.org/I1174212"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2045,"currency":"USD","value_usd":2045},"apc_paid":null,"fwci":9.3802,"has_fulltext":false,"cited_by_count":147,"citation_normalized_percentile":{"value":0.98607646,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"1","issue":"1","first_page":"84","last_page":"105"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9911999702453613,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9911999702453613,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.007199999876320362,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/shallow-neural-networks","display_name":"shallow neural networks","score":0.9150000214576721},{"id":"https://openalex.org/keywords/global-convergence","display_name":"global convergence","score":0.7770000100135803},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"stochastic gradient descent","score":0.7289999723434448},{"id":"https://openalex.org/keywords/random-initialization","display_name":"random initialization","score":0.7059999704360962},{"id":"https://openalex.org/keywords/interpolation","display_name":"interpolation","score":0.5360000133514404},{"id":"https://openalex.org/keywords/geometric-convergence-rate","display_name":"geometric convergence rate","score":0.6639999747276306},{"id":"https://openalex.org/keywords/nonconvex-optimization","display_name":"non-convex optimization","score":0.7300000190734863},{"id":"https://openalex.org/keywords/relu","display_name":"ReLU","score":0.5429999828338623}],"concepts":[{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.7410242557525635},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.7014251351356506},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6147066354751587},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.5791203379631042},{"id":"https://openalex.org/C141934464","wikidata":"https://www.wikidata.org/wiki/Q3305386","display_name":"Local optimum","level":2,"score":0.551295816898346},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.5402000546455383},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.4845830202102661},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.47078341245651245},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.45595884323120117},{"id":"https://openalex.org/C155032097","wikidata":"https://www.wikidata.org/wiki/Q798503","display_name":"Backpropagation","level":3,"score":0.4382507801055908},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.42619168758392334},{"id":"https://openalex.org/C2777027219","wikidata":"https://www.wikidata.org/wiki/Q1284190","display_name":"Constant (computer programming)","level":2,"score":0.42515137791633606},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.4100790023803711},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.36405038833618164},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.3519781529903412},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.33044230937957764},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3293481171131134},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.12679076194763184},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.08616036176681519},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/jsait.2020.2991332","doi":"https://doi.org/10.1109/jsait.2020.2991332","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jsait.2020.2991332","pdf_url":null,"source":{"id":"https://openalex.org/S4210211895","display_name":"IEEE Journal on Selected Areas in Information Theory","issn_l":"2641-8770","issn":["2641-8770"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-b","jufo-1","norway-1"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal on Selected Areas in Information Theory","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2520208575","display_name":null,"funder_award_id":"#FA9550-18-1-0078","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G3677612960","display_name":"CAREER: Guaranteed Nonconvex Optimization for High-Dimensional Learning","funder_award_id":"1846369","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G433588577","display_name":null,"funder_award_id":"CNS-1932254","funder_id":"https://openalex.org/F4320335353","funder_display_name":"National Science Foundation of Sri Lanka"},{"id":"https://openalex.org/G7011083491","display_name":"CIF: Small: Precise Computational and Statistical Tradeoffs for Iterative Signal Estimation and Supervised Learning","funder_award_id":"1813877","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306079","display_name":"David and Lucile Packard Foundation","ror":"https://ror.org/032atxq54"},{"id":"https://openalex.org/F4320309327","display_name":"Google","ror":"https://ror.org/00njsd438"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320335353","display_name":"National Science Foundation of Sri Lanka","ror":"https://ror.org/010xaa060"},{"id":"https://openalex.org/F4320338279","display_name":"Air Force Office of Scientific Research","ror":"https://ror.org/011e9bt93"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":57,"referenced_works":["https://openalex.org/W1573820523","https://openalex.org/W1678167890","https://openalex.org/W2000985550","https://openalex.org/W2399994860","https://openalex.org/W2591714514","https://openalex.org/W2619069683","https://openalex.org/W2625063094","https://openalex.org/W2626325961","https://openalex.org/W2709553318","https://openalex.org/W2739160792","https://openalex.org/W2741952635","https://openalex.org/W2765861987","https://openalex.org/W2766371994","https://openalex.org/W2777138330","https://openalex.org/W2785749378","https://openalex.org/W2787132992","https://openalex.org/W2787348029","https://openalex.org/W2789138075","https://openalex.org/W2798986185","https://openalex.org/W2804589149","https://openalex.org/W2810365967","https://openalex.org/W2886067286","https://openalex.org/W2889237180","https://openalex.org/W2889760744","https://openalex.org/W2891057995","https://openalex.org/W2894201461","https://openalex.org/W2894604724","https://openalex.org/W2899748887","https://openalex.org/W2899790086","https://openalex.org/W2900103278","https://openalex.org/W2900959181","https://openalex.org/W2904838594","https://openalex.org/W2907127169","https://openalex.org/W2942052807","https://openalex.org/W2949568260","https://openalex.org/W2949960976","https://openalex.org/W2950220847","https://openalex.org/W2962698540","https://openalex.org/W2963008903","https://openalex.org/W2963095610","https://openalex.org/W2963100491","https://openalex.org/W2963122491","https://openalex.org/W2963376662","https://openalex.org/W2963383839","https://openalex.org/W2963622172","https://openalex.org/W2963742538","https://openalex.org/W2963837241","https://openalex.org/W2963959597","https://openalex.org/W2964161337","https://openalex.org/W2970330753","https://openalex.org/W2970443625","https://openalex.org/W3093329015","https://openalex.org/W3104969455","https://openalex.org/W3119586787","https://openalex.org/W4297813530","https://openalex.org/W4302027967","https://openalex.org/W6634225129"],"related_works":["https://openalex.org/W4386939572","https://openalex.org/W2115605526","https://openalex.org/W3093883775","https://openalex.org/W1539246760","https://openalex.org/W2425229278","https://openalex.org/W2786746258","https://openalex.org/W4225893763","https://openalex.org/W2788727425","https://openalex.org/W4226299596","https://openalex.org/W4287868543"],"abstract_inverted_index":{"Many":[0],"modern":[1],"neural":[2,26,92,196],"network":[3,27,230],"architectures":[4,28],"are":[5],"trained":[6],"in":[7,29,103,125,138],"an":[8],"overparameterized":[9,25,141],"regime":[10],"where":[11,94],"the":[12,15,18,21,32,45,50,95,104,107,147,153,158,161,175,224,227,233,236],"parameters":[13,156,231],"of":[14,20,38,49,61,82,97,106,131,155,160,226,229,235],"model":[16],"exceed":[17,157],"size":[19,105,159,234],"training":[22,51,108,148,162,237],"dataset.":[23],"Sufficiently":[24],"principle":[30],"have":[31,86],"capacity":[33],"to":[34,69,71,120,134,144,216,250],"fit":[35],"any":[36,78],"set":[37],"labels":[39],"including":[40],"random":[41,115,210],"noise.":[42],"However,":[43,124],"given":[44],"highly":[46],"nonconvex":[47],"nature":[48],"landscape":[52],"it":[53],"is":[54,63,100,170],"not":[55],"clear":[56],"what":[57],"level":[58],"and":[59,137,179,198,248],"kind":[60],"overparameterization":[62,132],"required":[64],"for":[65,89,252],"first":[66],"order":[67],"methods":[68],"converge":[70,119],"a":[72,114,121,165,171,187,213,217,244],"global":[73,122,219],"optima":[74,220],"that":[75,88,203],"perfectly":[76,145],"interpolate":[77,146],"labels.":[79],"A":[80],"number":[81,96,154,228],"recent":[83],"theoretical":[84,177],"works":[85],"shown":[87],"very":[90],"wide":[91],"networks":[93],"hidden":[98],"units":[99],"polynomially":[101],"large":[102],"data":[109,149,163],"gradient":[110,205],"descent":[111,206],"starting":[112],"from":[113,243],"initialization":[116],"does":[117],"indeed":[118],"optima.":[123],"practice":[126],"much":[127],"more":[128],"moderate":[129],"levels":[130],"seems":[133],"be":[135],"sufficient":[136],"many":[139],"cases":[140],"models":[142],"seem":[143],"as":[150,152,221,223,256],"soon":[151,222],"by":[164],"constant":[166],"factor.":[167],"Thus":[168],"there":[169],"huge":[172],"gap":[173],"between":[174],"existing":[176],"literature":[178],"practical":[180],"experiments.":[181],"In":[182],"this":[183,191],"paper":[184],"we":[185,201],"take":[186],"step":[188],"towards":[189],"closing":[190],"gap.":[192],"Focusing":[193],"on":[194],"shallow":[195],"nets":[197],"smooth":[199],"activations,":[200],"show":[202],"(stochastic)":[204],"when":[207],"initialized":[208],"at":[209,212],"converges":[211],"geometric":[214],"rate":[215,247],"nearby":[218],"square-root":[225],"exceeds":[232],"data.":[238],"Our":[239],"results":[240],"also":[241],"benefit":[242],"fast":[245],"convergence":[246],"continue":[249],"hold":[251],"non-differentiable":[253],"activations":[254],"such":[255],"Rectified":[257],"Linear":[258],"Units":[259],"(ReLUs).":[260]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":21},{"year":2023,"cited_by_count":13},{"year":2022,"cited_by_count":14},{"year":2021,"cited_by_count":41},{"year":2020,"cited_by_count":33},{"year":2019,"cited_by_count":8},{"year":2018,"cited_by_count":2}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2025-10-10T00:00:00"}