{"id":"https://openalex.org/W2970176397","doi":"https://doi.org/10.48550/arxiv.1810.05369","title":"Regularization Matters: Generalization and Optimization of Neural Nets v.s. their Induced Kernel","display_name":"Regularization Matters: Generalization and Optimization of Neural Nets v.s. their Induced Kernel","alternate_titles":[],"publication_year":2018,"publication_date":"2018-10-12","ids":{"openalex":"https://openalex.org/W2970176397","doi":"https://doi.org/10.48550/arxiv.1810.05369","mag":"2970176397"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:1810.05369","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1810.05369","pdf_url":"https://arxiv.org/pdf/1810.05369","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W2970176397"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1810.05369","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113907654","display_name":"Colin Wei","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Wei, Colin","raw_affiliation_strings":["Stanford University ()"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University ()","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059740024","display_name":"Jason D. Lee","orcid":"https://orcid.org/0000-0003-0064-7800","observed_orcids":["https://orcid.org/0000-0003-0064-7800"]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Lee, Jason D.","raw_affiliation_strings":["Princeton University ,","Electrical Engineering"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton University ,","institution_ids":["https://openalex.org/I20089843"]},{"raw_affiliation_string":"Electrical Engineering","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Qiang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Liu, Qiang","raw_affiliation_strings":["University of Texas at Austin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101821970","display_name":"Tengyu Ma","orcid":"https://orcid.org/0009-0002-0677-4413","observed_orcids":["https://orcid.org/0009-0002-0677-4413"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Ma, Tengyu","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":172,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"32","issue":null,"first_page":"9709","last_page":"9721"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9688000082969666,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9688000082969666,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.02070000022649765,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.0031999999191612005,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/neural-tangent-kernel","display_name":"neural tangent kernel","score":0.921999990940094},{"id":"https://openalex.org/keywords/tikhonov-regularization","display_name":"Tikhonov regularization","score":0.7829999923706055},{"id":"https://openalex.org/keywords/generalization","display_name":"generalization","score":0.597000002861023},{"id":"https://openalex.org/keywords/sample-efficiency","display_name":"sample efficiency","score":0.6949999928474426},{"id":"https://openalex.org/keywords/lower-bounds","display_name":"lower bounds","score":0.5730000138282776},{"id":"https://openalex.org/keywords/kernel-methods","display_name":"kernel methods","score":0.8629999756813049},{"id":"https://openalex.org/keywords/relu-neural-networks","display_name":"ReLU networks","score":0.6859999895095825},{"id":"https://openalex.org/keywords/infinite-width-neural-networks","display_name":"infinite-width neural networks","score":0.753000020980835},{"id":"https://openalex.org/keywords/global-convergence","display_name":"global convergence","score":0.4690000116825104}],"concepts":[{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.5951366424560547},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5505986213684082},{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.5264878273010254},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5231448411941528},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.510459303855896},{"id":"https://openalex.org/C80884492","wikidata":"https://www.wikidata.org/wiki/Q3345678","display_name":"Reproducing kernel Hilbert space","level":3,"score":0.4985167980194092},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4733438789844513},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.4511238932609558},{"id":"https://openalex.org/C122280245","wikidata":"https://www.wikidata.org/wiki/Q620622","display_name":"Kernel method","level":3,"score":0.44104188680648804},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.42161253094673157},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.4003294110298157},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.38376104831695557},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.36072126030921936},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.28208377957344055},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.1760711371898651},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.1689300239086151},{"id":"https://openalex.org/C62799726","wikidata":"https://www.wikidata.org/wiki/Q190056","display_name":"Hilbert space","level":2,"score":0.0966390073299408},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.08582639694213867},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:1810.05369","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1810.05369","pdf_url":"https://arxiv.org/pdf/1810.05369","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.1810.05369","doi":"https://doi.org/10.48550/arxiv.1810.05369","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1810.05369","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"mag:2970176397","doi":null,"is_oa":false,"landing_page_url":"https://papers.nips.cc/paper/2019/file/8744cf92c88433f8cb04a02e6db69a0d-Paper.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":"https://openalex.org/P4404678063","host_organization_name":"Neural Information Processing Systems Foundation","host_organization_lineage":["https://openalex.org/P4404678063"],"host_organization_lineage_names":["Neural Information Processing Systems Foundation"],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"neural information processing systems","raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1810.05369","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1810.05369","pdf_url":"https://arxiv.org/pdf/1810.05369","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G7452299184","display_name":null,"funder_award_id":"W911NF","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G862988727","display_name":null,"funder_award_id":"W911NF-11","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306078","display_name":"U.S. Department of Defense","ror":"https://ror.org/0447fe631"},{"id":"https://openalex.org/F4320333591","display_name":"Multidisciplinary University Research Initiative","ror":null},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"},{"id":"https://openalex.org/F4320337003","display_name":"Office of Defense Programs","ror":"https://ror.org/0447fe631"},{"id":"https://openalex.org/F4320338281","display_name":"Army Research Office","ror":"https://ror.org/05epdh915"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2970176397.pdf","grobid_xml":"https://content.openalex.org/works/W2970176397.grobid-xml"},"referenced_works_count":83,"referenced_works":["https://openalex.org/W577198184","https://openalex.org/W813605148","https://openalex.org/W1509722456","https://openalex.org/W1544131626","https://openalex.org/W1553598118","https://openalex.org/W1608733719","https://openalex.org/W1720114023","https://openalex.org/W1811750039","https://openalex.org/W1891181203","https://openalex.org/W2004915807","https://openalex.org/W2032291279","https://openalex.org/W2107822587","https://openalex.org/W2119073761","https://openalex.org/W2127265926","https://openalex.org/W2130698119","https://openalex.org/W2143859313","https://openalex.org/W2160354932","https://openalex.org/W2167967601","https://openalex.org/W2168885649","https://openalex.org/W2243086562","https://openalex.org/W2399994860","https://openalex.org/W2519388618","https://openalex.org/W2552194003","https://openalex.org/W2593958421","https://openalex.org/W2604117713","https://openalex.org/W2608609325","https://openalex.org/W2618069152","https://openalex.org/W2626017178","https://openalex.org/W2626325961","https://openalex.org/W2709553318","https://openalex.org/W2739160792","https://openalex.org/W2741952635","https://openalex.org/W2765861987","https://openalex.org/W2766678531","https://openalex.org/W2766965791","https://openalex.org/W2768927505","https://openalex.org/W2774044172","https://openalex.org/W2777138330","https://openalex.org/W2785648721","https://openalex.org/W2785749378","https://openalex.org/W2787348029","https://openalex.org/W2788067318","https://openalex.org/W2788800397","https://openalex.org/W2788997738","https://openalex.org/W2789048837","https://openalex.org/W2789756211","https://openalex.org/W2790253170","https://openalex.org/W2792285703","https://openalex.org/W2798986185","https://openalex.org/W2804589149","https://openalex.org/W2805926765","https://openalex.org/W2807299122","https://openalex.org/W2809090039","https://openalex.org/W2886067286","https://openalex.org/W2889237180","https://openalex.org/W2894604724","https://openalex.org/W2895982930","https://openalex.org/W2899748887","https://openalex.org/W2899790086","https://openalex.org/W2900103278","https://openalex.org/W2900959181","https://openalex.org/W2904838594","https://openalex.org/W2910655610","https://openalex.org/W2911867426","https://openalex.org/W2912321142","https://openalex.org/W2913473169","https://openalex.org/W2926417365","https://openalex.org/W2927724204","https://openalex.org/W2942052807","https://openalex.org/W2950220847","https://openalex.org/W2952204734","https://openalex.org/W2962698540","https://openalex.org/W2963095610","https://openalex.org/W2963323437","https://openalex.org/W2963739978","https://openalex.org/W2964137095","https://openalex.org/W2964915616","https://openalex.org/W2971069898","https://openalex.org/W2973737673","https://openalex.org/W2996822312","https://openalex.org/W4286717651","https://openalex.org/W4294205985","https://openalex.org/W6920666171"],"related_works":["https://openalex.org/W2785619321","https://openalex.org/W4393190247","https://openalex.org/W3092390915","https://openalex.org/W3042003204","https://openalex.org/W3039141380","https://openalex.org/W2938647293","https://openalex.org/W7133186383","https://openalex.org/W4416955747","https://openalex.org/W4320853952","https://openalex.org/W2970239942"],"abstract_inverted_index":{"Recent":[0],"works":[1],"have":[2,51],"shown":[3],"that":[4,56,110,149,173],"on":[5,62,147,154],"sufficiently":[6],"over-parametrized":[7],"neural":[8,80,127,184],"nets,":[9,107,128,177],"gradient":[10,179],"descent":[11,180],"with":[12,83],"relatively":[13],"large":[14],"initialization":[15],"optimizes":[16,181],"a":[17,43,70,115,135,188],"prediction":[18],"function":[19],"in":[20,52,73,191],"the":[21,24,63,66,77,87,111,120,150,165,182],"RKHS":[22],"of":[23,65,114],"Neural":[25],"Tangent":[26],"Kernel":[27],"(NTK).":[28],"This":[29],"analysis":[30,100],"leads":[31],"to":[32,50,92,187],"global":[33,112,167,189],"convergence":[34],"results":[35],"but":[36,86],"does":[37],"not":[38],"work":[39],"when":[40],"there":[41],"is":[42,48,119,169],"standard":[44],"$\\ell_2$":[45],"regularizer,":[46],"which":[47,76,129,145],"useful":[49],"practice.":[53],"We":[54,171],"show":[55,109],"sample":[57],"efficiency":[58],"can":[59],"indeed":[60],"depend":[61],"presence":[64],"regularizer:":[67],"we":[68,97,108,133,162],"construct":[69],"simple":[71],"distribution":[72],"d":[74],"dimensions":[75],"optimal":[78],"regularized":[79,166,183],"net":[81,185],"learns":[82],"$O(d)$":[84],"samples":[85,91],"NTK":[88],"requires":[89],"$\u03a9(d^2)$":[90],"learn.":[93],"To":[94],"prove":[95,172],"this,":[96],"establish":[98],"two":[99],"tools:":[101],"i)":[102],"for":[103,138,142,174],"multi-layer":[104],"feedforward":[105],"ReLU":[106],"minimizer":[113],"weakly-regularized":[116],"cross-entropy":[117],"loss":[118,186],"max":[121],"normalized":[122],"margin":[123],"solution":[124],"among":[125],"all":[126],"generalizes":[130],"well;":[131],"ii)":[132],"develop":[134],"new":[136],"technique":[137],"proving":[139],"lower":[140],"bounds":[141],"kernel":[143,151],"methods,":[144],"relies":[146],"showing":[148],"cannot":[152],"focus":[153],"informative":[155],"features.":[156],"Motivated":[157],"by":[158],"our":[159],"generalization":[160],"results,":[161],"study":[163],"whether":[164],"optimum":[168],"attainable.":[170],"infinite-width":[175],"two-layer":[176],"noisy":[178],"minimum":[190],"polynomial":[192],"iterations.":[193]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2970176397","counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":16},{"year":2023,"cited_by_count":14},{"year":2022,"cited_by_count":21},{"year":2021,"cited_by_count":36},{"year":2020,"cited_by_count":35},{"year":2019,"cited_by_count":31},{"year":2018,"cited_by_count":2}],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2019-09-05T00:00:00"}
