{"id":"https://openalex.org/W4388090547","doi":"https://doi.org/10.48550/arxiv.2310.17813","title":"A Spectral Condition for Feature Learning","display_name":"A Spectral Condition for Feature Learning","publication_year":2023,"publication_date":"2023-10-26","ids":{"openalex":"https://openalex.org/W4388090547","doi":"https://doi.org/10.48550/arxiv.2310.17813"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2310.17813","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.17813","pdf_url":"https://arxiv.org/pdf/2310.17813","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2310.17813","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061115157","display_name":"Greg Yang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yang, Greg","raw_affiliation_strings":["UC Berkeley & Imbue"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UC Berkeley & Imbue","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013953784","display_name":"James B. Simon","orcid":"https://orcid.org/0000-0003-4210-0731","observed_orcids":["https://orcid.org/0000-0003-4210-0731"]},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Simon, James B.","raw_affiliation_strings":["UC Berkeley & Imbue"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UC Berkeley & Imbue","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5067399219","display_name":"Jeremy Bernstein","orcid":"https://orcid.org/0000-0001-9110-7476","observed_orcids":["https://orcid.org/0000-0001-9110-7476"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bernstein, Jeremy","raw_affiliation_strings":["MIT"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT","institution_ids":["https://openalex.org/I63966007"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":132,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.6265000104904175,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.6265000104904175,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.3537999987602234,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.00559999980032444,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/representation-learning","display_name":"representation learning","score":0.9819999933242798},{"id":"https://openalex.org/keywords/spectral-norm","display_name":"spectral norm","score":0.628000020980835},{"id":"https://openalex.org/keywords/weight-initialization","display_name":"weight initialization","score":0.7670000195503235},{"id":"https://openalex.org/keywords/spectral-scaling","display_name":"spectral scaling","score":0.6370000243186951},{"id":"https://openalex.org/keywords/gradient-updates","display_name":"gradient updates","score":0.3930000066757202},{"id":"https://openalex.org/keywords/internal-representations","display_name":"internal representations","score":0.4620000123977661},{"id":"https://openalex.org/keywords/large-scale-neural-networks","display_name":"large-scale neural networks","score":0.38499999046325684}],"concepts":[{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.7801222801208496},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.7485857009887695},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.6155140399932861},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.5843010544776917},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5263717770576477},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5016834735870361},{"id":"https://openalex.org/C191795146","wikidata":"https://www.wikidata.org/wiki/Q3878446","display_name":"Norm (philosophy)","level":2,"score":0.48720282316207886},{"id":"https://openalex.org/C202887219","wikidata":"https://www.wikidata.org/wiki/Q3895221","display_name":"Parametrization (atmospheric modeling)","level":3,"score":0.4734746217727661},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4194410443305969},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4064542353153229},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.37133675813674927},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.34857022762298584},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.2576178312301636},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0896587073802948},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.07355040311813354},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0653822124004364},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C74902906","wikidata":"https://www.wikidata.org/wiki/Q1190858","display_name":"Radiative transfer","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2310.17813","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.17813","pdf_url":"https://arxiv.org/pdf/2310.17813","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.17813","doi":"https://doi.org/10.48550/arxiv.2310.17813","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.17813","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2310.17813","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.17813","pdf_url":"https://arxiv.org/pdf/2310.17813","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2966665750","display_name":"Graduate Research Fellowship Program (GRFP)","funder_award_id":"1752814","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4388090547.pdf","grobid_xml":"https://content.openalex.org/works/W4388090547.grobid-xml"},"referenced_works_count":9,"referenced_works":["https://openalex.org/W2786602455","https://openalex.org/W3002073503","https://openalex.org/W3037932933","https://openalex.org/W4224035735","https://openalex.org/W4283331875","https://openalex.org/W4305004874","https://openalex.org/W4362707004","https://openalex.org/W4365211469","https://openalex.org/W4385964757"],"related_works":["https://openalex.org/W7133184588","https://openalex.org/W7162402950","https://openalex.org/W7112197770","https://openalex.org/W7133224629","https://openalex.org/W4415797066","https://openalex.org/W4320340812","https://openalex.org/W7009222383","https://openalex.org/W4415337542","https://openalex.org/W4416048536","https://openalex.org/W4396600916"],"abstract_inverted_index":{"The":[0],"push":[1],"to":[2,24,67,86,99],"train":[3],"ever":[4],"larger":[5],"neural":[6,112],"networks":[7],"has":[8],"motivated":[9],"the":[10,54,101],"study":[11],"of":[12,57,90,108],"initialization":[13],"and":[14,60,77],"training":[15,26],"at":[16,35],"large":[17],"network":[18],"width.":[19],"A":[20],"key":[21],"challenge":[22],"is":[23,50],"scale":[25],"so":[27],"that":[28,47],"a":[29,38,104],"network's":[30],"internal":[31],"representations":[32],"evolve":[33],"nontrivially":[34],"all":[36],"widths,":[37],"process":[39],"known":[40],"as":[41],"feature":[42,48,109],"learning.":[43],"Here,":[44],"we":[45,97],"show":[46],"learning":[49,110],"achieved":[51],"by":[52],"scaling":[53,82],"spectral":[55,81],"norm":[56,76],"weight":[58],"matrices":[59],"their":[61],"updates":[62],"like":[63],"$\\sqrt{\\texttt{fan-out}/\\texttt{fan-in}}$,":[64],"in":[65,95,111],"contrast":[66],"widely":[68],"used":[69],"but":[70],"heuristic":[71],"scalings":[72],"based":[73],"on":[74],"Frobenius":[75],"entry":[78],"size.":[79],"Our":[80],"analysis":[83],"also":[84],"leads":[85],"an":[87],"elementary":[88],"derivation":[89],"\\emph{maximal":[91],"update":[92],"parametrization}.":[93],"All":[94],"all,":[96],"aim":[98],"provide":[100],"reader":[102],"with":[103],"solid":[105],"conceptual":[106],"understanding":[107],"networks.":[113]},"counts_by_year":[{"year":2026,"cited_by_count":68},{"year":2025,"cited_by_count":41},{"year":2024,"cited_by_count":21},{"year":2023,"cited_by_count":1},{"year":2020,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2023-11-01T00:00:00"}