{"id":"https://openalex.org/W2913473169","doi":"https://doi.org/10.48550/arxiv.1902.04760","title":"Scaling Limits of Wide Neural Networks with Weight Sharing: Gaussian Process Behavior, Gradient Independence, and Neural Tangent Kernel Derivation","display_name":"Scaling Limits of Wide Neural Networks with Weight Sharing: Gaussian Process Behavior, Gradient Independence, and Neural Tangent Kernel Derivation","alternate_titles":[],"publication_year":2019,"publication_date":"2019-02-13","ids":{"openalex":"https://openalex.org/W2913473169","doi":"https://doi.org/10.48550/arxiv.1902.04760","mag":"2913473169"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:1902.04760","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1902.04760","pdf_url":"https://arxiv.org/pdf/1902.04760","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W2913473169"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1902.04760","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061115157","display_name":"Greg Yang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"contributor_roles":[],"raw_author_name":"Yang, Greg","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5061115157"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":334,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12814","display_name":"Gaussian Processes and Bayesian Inference","score":0.6000999808311462,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12814","display_name":"Gaussian Processes and Bayesian Inference","score":0.6000999808311462,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.2281000018119812,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.1370999962091446,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/wide-neural-networks","display_name":"wide neural networks","score":0.7509999871253967},{"id":"https://openalex.org/keywords/scaling-limits","display_name":"scaling limits","score":0.9350000023841858},{"id":"https://openalex.org/keywords/gaussian-processes","display_name":"Gaussian processes","score":0.6190000176429749},{"id":"https://openalex.org/keywords/neural-tangent-kernel","display_name":"neural tangent kernel","score":0.8220000267028809},{"id":"https://openalex.org/keywords/random-matrix-theory","display_name":"random matrix theory","score":0.675000011920929},{"id":"https://openalex.org/keywords/semicircle-law","display_name":"semicircle law","score":0.8679999709129333},{"id":"https://openalex.org/keywords/marchenko-pastur-law","display_name":"Marchenko-Pastur law","score":0.9980000257492065},{"id":"https://openalex.org/keywords/recurrent-neural-networks","display_name":"recurrent neural networks","score":0.7860000133514404},{"id":"https://openalex.org/keywords/convolutional-neural-networks","display_name":"convolutional neural networks","score":0.9750000238418579},{"id":"https://openalex.org/keywords/resnet","display_name":"ResNet","score":0.9800000190734863},{"id":"https://openalex.org/keywords/attention","display_name":"attention","score":0.6449999809265137},{"id":"https://openalex.org/keywords/batch-normalization","display_name":"batch normalization","score":0.9290000200271606},{"id":"https://openalex.org/keywords/initialization-scheme","display_name":"initialization scheme","score":0.37700000405311584}],"concepts":[{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6935209631919861},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.6702152490615845},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.5647723078727722},{"id":"https://openalex.org/C61326573","wikidata":"https://www.wikidata.org/wiki/Q1496376","display_name":"Gaussian process","level":3,"score":0.5282230377197266},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5070376396179199},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.49036622047424316},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.43907055258750916},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.4238809049129486},{"id":"https://openalex.org/C155032097","wikidata":"https://www.wikidata.org/wiki/Q798503","display_name":"Backpropagation","level":3,"score":0.41665780544281006},{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.4159236550331116},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.4030718207359314},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.37885355949401855},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3766632378101349},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32347172498703003},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.11349627375602722},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.10300394892692566},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.08529722690582275},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:1902.04760","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1902.04760","pdf_url":"https://arxiv.org/pdf/1902.04760","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.1902.04760","doi":"https://doi.org/10.48550/arxiv.1902.04760","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1902.04760","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1902.04760","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1902.04760","pdf_url":"https://arxiv.org/pdf/1902.04760","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2913473169.pdf","grobid_xml":"https://content.openalex.org/works/W2913473169.grobid-xml"},"referenced_works_count":87,"referenced_works":["https://openalex.org/W602904462","https://openalex.org/W1493832124","https://openalex.org/W1533861849","https://openalex.org/W1567512734","https://openalex.org/W1625958017","https://openalex.org/W1677182931","https://openalex.org/W1836465849","https://openalex.org/W1866206747","https://openalex.org/W1891181203","https://openalex.org/W1928278792","https://openalex.org/W1950611080","https://openalex.org/W1959879694","https://openalex.org/W1987772002","https://openalex.org/W1989774907","https://openalex.org/W2008448456","https://openalex.org/W2027802883","https://openalex.org/W2064675550","https://openalex.org/W2066350599","https://openalex.org/W2069129925","https://openalex.org/W2082029531","https://openalex.org/W2090878544","https://openalex.org/W2108394050","https://openalex.org/W2112796928","https://openalex.org/W2130109580","https://openalex.org/W2133564696","https://openalex.org/W2154087390","https://openalex.org/W2157331557","https://openalex.org/W2167085024","https://openalex.org/W2167608136","https://openalex.org/W2168248818","https://openalex.org/W2194775991","https://openalex.org/W2212676342","https://openalex.org/W2423689290","https://openalex.org/W2549019841","https://openalex.org/W2556364298","https://openalex.org/W2565889208","https://openalex.org/W2610971674","https://openalex.org/W2626778328","https://openalex.org/W2745165998","https://openalex.org/W2747181576","https://openalex.org/W2752851182","https://openalex.org/W2760969859","https://openalex.org/W2761315682","https://openalex.org/W2766678531","https://openalex.org/W2787173309","https://openalex.org/W2793904650","https://openalex.org/W2803439868","https://openalex.org/W2807037403","https://openalex.org/W2808197691","https://openalex.org/W2808460953","https://openalex.org/W2809090039","https://openalex.org/W2885550588","https://openalex.org/W2888674590","https://openalex.org/W2889737445","https://openalex.org/W2890473586","https://openalex.org/W2894604724","https://openalex.org/W2895016186","https://openalex.org/W2898146813","https://openalex.org/W2898324627","https://openalex.org/W2899205164","https://openalex.org/W2899748887","https://openalex.org/W2899790086","https://openalex.org/W2900103278","https://openalex.org/W2900959181","https://openalex.org/W2907747478","https://openalex.org/W2910655610","https://openalex.org/W2920270464","https://openalex.org/W2949117887","https://openalex.org/W2950139980","https://openalex.org/W2950635152","https://openalex.org/W2950774935","https://openalex.org/W2952861622","https://openalex.org/W2962698540","https://openalex.org/W2962723954","https://openalex.org/W2962939986","https://openalex.org/W2962941879","https://openalex.org/W2963323437","https://openalex.org/W2963446712","https://openalex.org/W2963570896","https://openalex.org/W2964067469","https://openalex.org/W2964083594","https://openalex.org/W2996822312","https://openalex.org/W3031355122","https://openalex.org/W3037932933","https://openalex.org/W3116019732","https://openalex.org/W4294205985","https://openalex.org/W7045062981"],"related_works":["https://openalex.org/W3038074040","https://openalex.org/W7167689354","https://openalex.org/W3178741258","https://openalex.org/W3180313059","https://openalex.org/W2912322140","https://openalex.org/W7038090036","https://openalex.org/W2970889419","https://openalex.org/W3132350540","https://openalex.org/W3164547988","https://openalex.org/W7083804515"],"abstract_inverted_index":{"Several":[0],"recent":[1,215],"trends":[2],"in":[3,131,141,187,217,249],"machine":[4],"learning":[5],"theory":[6],"and":[7,71,82,111,153,208,243],"practice,":[8],"from":[9,139],"the":[10,17,89,124,142,160,163,206,209],"design":[11,231],"of":[12,20,48,59,91,150,162,176,232,246],"state-of-the-art":[13],"Gaussian":[14,96,235],"Process":[15],"to":[16,34,41,95,136,147,172,198,239],"convergence":[18,90,161],"analysis":[19],"deep":[21],"neural":[22,38,68,93,103,106,177,218],"nets":[23],"(DNNs)":[24],"under":[25,122,179],"stochastic":[26],"gradient":[27,151,180,241],"descent":[28],"(SGD),":[29],"have":[30],"found":[31],"it":[32,156],"fruitful":[33],"study":[35],"wide":[36],"random":[37,92,201],"networks.":[39,50],"Central":[40],"these":[42,53],"approaches":[43],"are":[44,80],"certain":[45],"scaling":[46,75],"limits":[47],"such":[49,100,204],"We":[51,223],"unify":[52],"results":[54,203,216],"by":[55],"introducing":[56],"a":[57,60,167,228],"notion":[58],"straightline":[61],"\\emph{tensor":[62],"program}":[63],"that":[64,129],"can":[65,133],"express":[66],"most":[67],"network":[69,219],"computations,":[70],"we":[72],"characterize":[73],"its":[74,78],"limit":[76],"when":[77,155],"tensors":[79],"large":[81],"randomized.":[83],"From":[84],"our":[85,193,225],"framework":[86,194],"follows":[87],"(1)":[88,188],"networks":[94,178],"processes":[97],"for":[98,184],"architectures":[99,186],"as":[101,205,212,214],"recurrent":[102],"networks,":[104,107,109],"convolutional":[105],"residual":[108],"attention,":[110],"any":[112],"combination":[113],"thereof,":[114],"with":[115],"or":[116],"without":[117,189],"batch":[118,190],"normalization;":[119],"(2)":[120],"conditions":[121],"which":[123],"\\emph{gradient":[125],"independence":[126],"assumption}":[127],"--":[128,145],"weights":[130,140],"backpropagation":[132],"be":[134,137],"assumed":[135],"independent":[138],"forward":[143],"pass":[144],"leads":[146],"correct":[148],"computation":[149],"dynamics,":[152],"corrections":[154],"does":[157],"not;":[158],"(3)":[159],"Neural":[164],"Tangent":[165],"Kernel,":[166],"recently":[168],"proposed":[169],"kernel":[170],"used":[171],"predict":[173],"training":[174],"dynamics":[175,248],"descent,":[181],"at":[182],"initialization":[183,237],"all":[185],"normalization.":[191],"Mathematically,":[192],"is":[195],"general":[196],"enough":[197],"rederive":[199],"classical":[200],"matrix":[202],"semicircle":[207],"Marchenko-Pastur":[210],"laws,":[211],"well":[213],"Jacobian":[220],"singular":[221],"values.":[222],"hope":[224],"work":[226],"opens":[227],"way":[229],"toward":[230],"even":[233],"stronger":[234],"Processes,":[236],"schemes":[238],"avoid":[240],"explosion/vanishing,":[242],"deeper":[244],"understanding":[245],"SGD":[247],"modern":[250],"architectures.":[251]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2913473169","counts_by_year":[{"year":2026,"cited_by_count":25},{"year":2025,"cited_by_count":39},{"year":2024,"cited_by_count":33},{"year":2023,"cited_by_count":41},{"year":2022,"cited_by_count":41},{"year":2021,"cited_by_count":53},{"year":2020,"cited_by_count":61},{"year":2019,"cited_by_count":36},{"year":2018,"cited_by_count":5}],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2019-02-21T00:00:00"}
