{"id":"https://openalex.org/W6996442111","doi":null,"title":"SGD and Weight Decay Provably Induce a Low-Rank Bias in Deep Neural Networks","display_name":"SGD and Weight Decay Provably Induce a Low-Rank Bias in Deep Neural Networks","publication_year":2023,"publication_date":"2023-02-14","ids":{"openalex":"https://openalex.org/W6996442111"},"language":"en","primary_location":{"id":"pmh:oai:dspace.mit.edu:1721.1/148231","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/1721.1/148231","pdf_url":"https://dspace.mit.edu/bitstream/1721.1/148231/1/CBMM-Memo-140.pdf","source":{"id":"https://openalex.org/S4306400425","display_name":"DSpace@MIT (Massachusetts Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I63966007","host_organization_name":"Massachusetts Institute of Technology","host_organization_lineage":["https://openalex.org/I63966007"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Technical Report"},"volume":null,"type":"report","is_traditional":false,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://dspace.mit.edu/bitstream/1721.1/148231/1/CBMM-Memo-140.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050776111","display_name":"Tomer Galanti","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Galanti, Tomer","raw_affiliation_strings":[": Center for Brains, Minds, and Machines , MIT , Cambridge , MA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":": Center for Brains, Minds, and Machines , MIT , Cambridge , MA , USA","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100018314","display_name":"Zachary S. Siegel","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Siegel, Zachary","raw_affiliation_strings":[": Department of Computer Science , Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":": Department of Computer Science , Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015434599","display_name":"Aparna Gupte","orcid":"https://orcid.org/0009-0007-2809-1257","observed_orcids":["https://orcid.org/0009-0007-2809-1257"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gupte, Aparna","raw_affiliation_strings":[": Center for Brains, Minds, and Machines , MIT , Cambridge , MA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":": Center for Brains, Minds, and Machines , MIT , Cambridge , MA , USA","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001833084","display_name":"Tomaso Poggio","orcid":"https://orcid.org/0000-0002-3944-0455","observed_orcids":["https://orcid.org/0000-0002-3944-0455"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Poggio, Tomaso","raw_affiliation_strings":[": Center for Brains, Minds, and Machines , MIT , Cambridge , MA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":": Center for Brains, Minds, and Machines , MIT , Cambridge , MA , USA","institution_ids":["https://openalex.org/I63966007"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.0007999999797903001,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"stochastic gradient descent","score":0.9020000100135803},{"id":"https://openalex.org/keywords/weight-decay","display_name":"weight decay","score":1.0},{"id":"https://openalex.org/keywords/rank-minimization","display_name":"rank minimization","score":0.7979999780654907},{"id":"https://openalex.org/keywords/relu-neural-networks","display_name":"ReLU networks","score":0.9409999847412109},{"id":"https://openalex.org/keywords/mini-batch-size","display_name":"mini-batch size","score":0.6589999794960022},{"id":"https://openalex.org/keywords/learning-rate","display_name":"learning rate","score":0.9710000157356262},{"id":"https://openalex.org/keywords/generalization","display_name":"generalization","score":0.9509999752044678},{"id":"https://openalex.org/keywords/skip-connections","display_name":"skip connections","score":0.5289999842643738},{"id":"https://openalex.org/keywords/convolutional-layers","display_name":"convolutional layers","score":0.6520000100135803},{"id":"https://openalex.org/keywords/implicit-regularization","display_name":"implicit regularization","score":0.4959999918937683}],"concepts":[{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.7073000073432922},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.65420001745224},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.588699996471405},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4927999973297119},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49050000309944153},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48170000314712524},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4578000009059906},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.4244999885559082},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4090999960899353},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.40639999508857727},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.39899998903274536},{"id":"https://openalex.org/C13355873","wikidata":"https://www.wikidata.org/wiki/Q2920850","display_name":"Connection (principal bundle)","level":2,"score":0.38929998874664307},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.38510000705718994},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.35670000314712524},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.3416000008583069},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.31029999256134033},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.2727000117301941},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.2596000134944916}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:dspace.mit.edu:1721.1/148231","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/1721.1/148231","pdf_url":"https://dspace.mit.edu/bitstream/1721.1/148231/1/CBMM-Memo-140.pdf","source":{"id":"https://openalex.org/S4306400425","display_name":"DSpace@MIT (Massachusetts Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I63966007","host_organization_name":"Massachusetts Institute of Technology","host_organization_lineage":["https://openalex.org/I63966007"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Technical Report"}],"best_oa_location":{"id":"pmh:oai:dspace.mit.edu:1721.1/148231","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/1721.1/148231","pdf_url":"https://dspace.mit.edu/bitstream/1721.1/148231/1/CBMM-Memo-140.pdf","source":{"id":"https://openalex.org/S4306400425","display_name":"DSpace@MIT (Massachusetts Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I63966007","host_organization_name":"Massachusetts Institute of Technology","host_organization_lineage":["https://openalex.org/I63966007"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Technical Report"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2830697258","display_name":"Collaborative Proposal:   Object and Action Recognition in Time Sequences of Images:   Computational Neuroscience and Neurophysiology","funder_award_id":"0827427","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5998761286","display_name":"Computational Models and Physiological Studies of Feedback in Visual Object Recognition Tasks","funder_award_id":"0640097","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7201050375","display_name":"A Center for Brains, Minds and Machines: the Science and the Technology of Intelligence","funder_award_id":"1231216","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7809643752","display_name":null,"funder_award_id":"CCF-1231216","funder_id":"https://openalex.org/F4320324608","funder_display_name":"Companhia Brasileira de Metalurgia e Minera\u00e7\u00e3o"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320324608","display_name":"Companhia Brasileira de Metalurgia e Minera\u00e7\u00e3o","ror":"https://ror.org/03k8d9631"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W6996442111.pdf","grobid_xml":"https://content.openalex.org/works/W6996442111.grobid-xml"},"referenced_works_count":7,"referenced_works":["https://openalex.org/W2112796928","https://openalex.org/W2950220847","https://openalex.org/W3094502228","https://openalex.org/W3100332668","https://openalex.org/W3121845051","https://openalex.org/W3193421702","https://openalex.org/W4221165319"],"related_works":["https://openalex.org/W4298048409","https://openalex.org/W4403882521","https://openalex.org/W4282926156","https://openalex.org/W7133238145","https://openalex.org/W3126986115","https://openalex.org/W7048107489","https://openalex.org/W4414849807","https://openalex.org/W4391631942","https://openalex.org/W4315473667","https://openalex.org/W7102238376"],"abstract_inverted_index":{"In":[0],"this":[1,55,85,94],"paper,":[2],"we":[3,48,73,88],"study":[4],"the":[5,43,91],"bias":[6,38,56,95],"of":[7,115,122],"Stochastic":[8],"Gradient":[9],"Descent":[10],"(SGD)":[11],"to":[12,83,119],"learn":[13],"low-rank":[14],"weight":[15,34,44,70,79],"matrices":[16],"when":[17,60],"training":[18,27],"deep":[19],"ReLU":[20],"neural":[21,28,120],"networks.":[22],"Our":[23,107],"results":[24],"show":[25],"that":[26,54,78,99],"networks":[29,121],"with":[30,129],"mini-batch":[31],"SGD":[32],"and":[33,52,75,96,117,132],"decay":[35,80],"causes":[36],"a":[37,102,112],"towards":[39],"rank":[40],"minimization":[41],"over":[42],"matri-":[45],"ces.":[46],"Specifically,":[47],"show,":[49],"both":[50],"theoretically":[51],"empirically,":[53],"is":[57,81,109],"more":[58],"pronounced":[59],"using":[61],"smaller":[62],"batch":[63],"sizes,":[64],"higher":[65],"learning":[66],"rates,":[67],"or":[68,125],"increased":[69],"decay.":[71],"Additionally,":[72],"predict":[74],"observe":[76],"empirically":[77,89],"necessary":[82],"achieve":[84],"bias.":[86],"Finally,":[87],"investigate":[90],"connection":[92],"between":[93],"generalization,":[97],"finding":[98],"it":[100],"has":[101],"marginal":[103],"effect":[104],"on":[105,111],"generalization.":[106],"analysis":[108],"based":[110],"minimal":[113],"set":[114],"assumptions":[116],"applies":[118],"any":[123],"width":[124],"depth,":[126],"including":[127],"those":[128],"residual":[130],"connections":[131],"convolutional":[133],"layers.":[134]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-10T00:00:00"}