{"id":"https://openalex.org/W2949960976","doi":"https://doi.org/10.48550/arxiv.1906.05392","title":"Generalization Guarantees for Neural Networks via Harnessing the Low-rank Structure of the Jacobian","display_name":"Generalization Guarantees for Neural Networks via Harnessing the Low-rank Structure of the Jacobian","publication_year":2019,"publication_date":"2019-06-12","ids":{"openalex":"https://openalex.org/W2949960976","doi":"https://doi.org/10.48550/arxiv.1906.05392","mag":"2949960976"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:1906.05392","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1906.05392","pdf_url":"https://arxiv.org/pdf/1906.05392","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1906.05392","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050547472","display_name":"Samet Oymak","orcid":"https://orcid.org/0000-0001-5203-0752","observed_orcids":["https://orcid.org/0000-0001-5203-0752"]},"institutions":[{"id":"https://openalex.org/I103635307","display_name":"University of California, Riverside","ror":"https://ror.org/03nawhv43","country_code":"US","type":"education","lineage":["https://openalex.org/I103635307"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Oymak, Samet","raw_affiliation_strings":["Department of Electrical and Computer Engineering , University of California , Riverside , CA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering , University of California , Riverside , CA","institution_ids":["https://openalex.org/I103635307"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088599790","display_name":"Zalan Fabian","orcid":"https://orcid.org/0000-0002-6968-1233","observed_orcids":["https://orcid.org/0000-0002-6968-1233"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fabian, Zalan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074422516","display_name":"Mingchen Li","orcid":"https://orcid.org/0000-0002-0736-1541","observed_orcids":["https://orcid.org/0000-0002-0736-1541"]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li, Mingchen","raw_affiliation_strings":["Department of Electrical Engineering , University of Southern California , Los Angeles , CA","Equal contribution"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering , University of Southern California , Los Angeles , CA","institution_ids":["https://openalex.org/I1174212"]},{"raw_affiliation_string":"Equal contribution","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046962187","display_name":"Mahdi Soltanolkotabi","orcid":"https://orcid.org/0000-0003-2101-6418","observed_orcids":["https://orcid.org/0000-0003-2101-6418"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Soltanolkotabi, Mahdi","raw_affiliation_strings":["Ming Hsieh"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ming Hsieh","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":94,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9707000255584717,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9707000255584717,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.023800000548362732,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0015999999595806003,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/neural-network-generalization","display_name":"neural network generalization","score":0.9229999780654907},{"id":"https://openalex.org/keywords/singular-values","display_name":"singular values","score":0.6079999804496765},{"id":"https://openalex.org/keywords/noisy-labels","display_name":"label noise","score":0.6309999823570251},{"id":"https://openalex.org/keywords/early-stopping","display_name":"early stopping","score":0.8240000009536743},{"id":"https://openalex.org/keywords/label-alignment","display_name":"label alignment","score":0.515999972820282},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"gradient descent","score":0.6660000085830688}],"concepts":[{"id":"https://openalex.org/C200331156","wikidata":"https://www.wikidata.org/wiki/Q506041","display_name":"Jacobian matrix and determinant","level":2,"score":0.8994230628013611},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.8817273378372192},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.7483400702476501},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.45922815799713135},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.41534897685050964},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.40890151262283325},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3332122564315796},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3165287375450134},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.14765557646751404},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.1222427487373352},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.046607524156570435}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:1906.05392","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1906.05392","pdf_url":"https://arxiv.org/pdf/1906.05392","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.1906.05392","doi":"https://doi.org/10.48550/arxiv.1906.05392","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1906.05392","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1906.05392","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1906.05392","pdf_url":"https://arxiv.org/pdf/1906.05392","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2520208575","display_name":null,"funder_award_id":"#FA9550-18-1-0078","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G3045592441","display_name":null,"funder_award_id":"FA9550-18","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G3677612960","display_name":"CAREER: Guaranteed Nonconvex Optimization for High-Dimensional Learning","funder_award_id":"1846369","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7011083491","display_name":"CIF: Small: Precise Computational and Statistical Tradeoffs for Iterative Signal Estimation and Supervised Learning","funder_award_id":"1813877","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320338279","display_name":"Air Force Office of Scientific Research","ror":"https://ror.org/011e9bt93"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2949960976.pdf","grobid_xml":"https://content.openalex.org/works/W2949960976.grobid-xml"},"referenced_works_count":63,"referenced_works":["https://openalex.org/W1542886316","https://openalex.org/W1573820523","https://openalex.org/W1678167890","https://openalex.org/W1811750039","https://openalex.org/W2088911135","https://openalex.org/W2099579348","https://openalex.org/W2115881141","https://openalex.org/W2472904648","https://openalex.org/W2523060838","https://openalex.org/W2552194003","https://openalex.org/W2579923771","https://openalex.org/W2604117713","https://openalex.org/W2626325961","https://openalex.org/W2741952635","https://openalex.org/W2765861987","https://openalex.org/W2766965791","https://openalex.org/W2777138330","https://openalex.org/W2785631679","https://openalex.org/W2785749378","https://openalex.org/W2790253170","https://openalex.org/W2798418464","https://openalex.org/W2804589149","https://openalex.org/W2810365967","https://openalex.org/W2894604724","https://openalex.org/W2898193427","https://openalex.org/W2899748887","https://openalex.org/W2899790086","https://openalex.org/W2900103278","https://openalex.org/W2900959181","https://openalex.org/W2904838594","https://openalex.org/W2911867426","https://openalex.org/W2912321142","https://openalex.org/W2918775908","https://openalex.org/W2921467542","https://openalex.org/W2922153390","https://openalex.org/W2925181975","https://openalex.org/W2926417365","https://openalex.org/W2927724204","https://openalex.org/W2938647293","https://openalex.org/W2941057241","https://openalex.org/W2944862841","https://openalex.org/W2947989684","https://openalex.org/W2950220847","https://openalex.org/W2952204734","https://openalex.org/W2962933129","https://openalex.org/W2963095610","https://openalex.org/W2963100491","https://openalex.org/W2963518130","https://openalex.org/W2963655672","https://openalex.org/W2963739978","https://openalex.org/W2963794891","https://openalex.org/W2963826371","https://openalex.org/W2963837241","https://openalex.org/W2963959597","https://openalex.org/W2970330753","https://openalex.org/W2970618525","https://openalex.org/W2996822312","https://openalex.org/W3000593479","https://openalex.org/W3009072036","https://openalex.org/W3021189130","https://openalex.org/W3119586787","https://openalex.org/W4289345622","https://openalex.org/W6920666171"],"related_works":["https://openalex.org/W3013844214","https://openalex.org/W3003398363","https://openalex.org/W2753910463","https://openalex.org/W2529714286","https://openalex.org/W4405254712","https://openalex.org/W2786622092","https://openalex.org/W2084201419","https://openalex.org/W4416074406","https://openalex.org/W3206818904","https://openalex.org/W2938647293"],"abstract_inverted_index":{"Modern":[0],"neural":[1,26,199,229],"network":[2,80,173],"architectures":[3],"often":[4],"generalize":[5,141,203],"well":[6,76,178],"despite":[7],"containing":[8],"many":[9,241],"more":[10],"parameters":[11],"than":[12],"the":[13,16,22,42,46,51,79,83,86,101,111,121,144,158,167,172,179,185,225,252,261,274],"size":[14,81],"of":[15,25,45,85,160,171,191,227,260],"training":[17,58,136,147],"dataset.":[18],"This":[19],"paper":[20],"explores":[21],"generalization":[23,38,60,156,169],"capabilities":[24],"networks":[27,215,230],"trained":[28],"via":[29],"gradient":[30],"descent.":[31],"We":[32,117,163,208],"develop":[33],"a":[34,96,132,235,246],"data-dependent":[35],"optimization":[36],"and":[37,59,65,68,72,88,106,113,127,150,221,240,258,268,277],"theory":[39],"which":[40],"leverages":[41],"low-rank":[43,232],"structure":[44,233],"Jacobian":[47,102,226],"matrix":[48],"associated":[49,109],"with":[50,110,134,155,184,234],"network.":[52],"Our":[53],"results":[54,193],"help":[55,154],"demystify":[56],"why":[57],"is":[61,125,148,174,182,194,256],"easier":[62],"on":[63,70,213,265,273],"clean":[64],"structured":[66],"datasets":[67,74],"harder":[69],"noisy":[71],"unstructured":[73],"as":[75,77],"how":[78,177],"affects":[82],"evolution":[84],"train":[87,131],"test":[89],"errors":[90],"during":[91],"training.":[92],"Specifically,":[93],"we":[94],"use":[95],"control":[97],"knob":[98],"to":[99,245],"split":[100],"spectum":[103],"into":[104],"\"information\"":[105],"\"nuisance\"":[107],"spaces":[108],"large":[112,237],"small":[114,242],"singular":[115,238],"values.":[116],"show":[118,165],"that":[119,138,166,195,216],"over":[120,251],"information":[122,186,248,253],"space":[123,146,254,276],"learning":[124,255],"fast":[126,257],"one":[128],"can":[129,139,153,201],"quickly":[130],"model":[133],"zero":[135],"loss":[137],"also":[140,164],"well.":[142],"Over":[143],"nuisance":[145,275],"slower":[149],"early":[151],"stopping":[152],"at":[157],"expense":[159],"some":[161],"bias.":[162],"overall":[168],"capability":[170],"controlled":[175],"by":[176],"label":[180,262,270],"vector":[181,263],"aligned":[183],"space.":[187],"A":[188],"key":[189],"feature":[190],"our":[192,218],"even":[196],"constant":[197],"width":[198],"nets":[200],"provably":[202],"for":[204],"sufficiently":[205],"nice":[206],"datasets.":[207],"conduct":[209],"various":[210],"numerical":[211],"experiments":[212],"deep":[214],"corroborate":[217],"theoretical":[219],"findings":[220],"demonstrate":[222],"that:":[223],"(i)":[224],"typical":[228],"exhibit":[231],"few":[236],"values":[239],"ones":[243],"leading":[244],"low-dimensional":[247],"space,":[249,267],"(ii)":[250],"most":[259],"falls":[264,272],"this":[266],"(iii)":[269],"noise":[271],"impedes":[278],"optimization/generalization.":[279]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":17},{"year":2024,"cited_by_count":14},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":16},{"year":2020,"cited_by_count":14},{"year":2019,"cited_by_count":6}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2019-06-27T00:00:00"}