{"id":"https://openalex.org/W3133220799","doi":"https://doi.org/10.48550/arxiv.2105.01029","title":"Initialization and Regularization of Factorized Neural Layers","display_name":"Initialization and Regularization of Factorized Neural Layers","alternate_titles":[],"publication_year":2021,"publication_date":"2021-05-03","ids":{"openalex":"https://openalex.org/W3133220799","doi":"https://doi.org/10.48550/arxiv.2105.01029","mag":"3133220799"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2105.01029","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2105.01029","pdf_url":"https://arxiv.org/pdf/2105.01029","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W3133220799"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2105.01029","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005152458","display_name":"Mikhail Khodak","orcid":"https://orcid.org/0000-0003-0089-5361","observed_orcids":["https://orcid.org/0000-0003-0089-5361"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Khodak, Mikhail","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054432618","display_name":"Neil Tenenholtz","orcid":"https://orcid.org/0000-0003-1250-3716","observed_orcids":["https://orcid.org/0000-0003-1250-3716"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tenenholtz, Neil","raw_affiliation_strings":["Microsoft Research","Microsoft Research#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft Research#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018151279","display_name":"Lester W. Mackey","orcid":"https://orcid.org/0000-0002-1102-0387","observed_orcids":["https://orcid.org/0000-0002-1102-0387"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4400600948","display_name":"Microsoft Research New England (United States)","ror":"https://ror.org/010wkqn50","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4400600948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mackey, Lester","raw_affiliation_strings":["Microsoft Research","Microsoft Research New, England#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft Research New, England#TAB#","institution_ids":["https://openalex.org/I4400600948"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021028602","display_name":"Nicol\u00f2 Fusi","orcid":"https://orcid.org/0000-0002-4102-0169","observed_orcids":["https://orcid.org/0000-0002-4102-0169"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fusi, Nicol\u00f2","raw_affiliation_strings":["Microsoft Research","(Microsoft)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"(Microsoft)","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":49,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.8241999745368958,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.8241999745368958,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.12919999659061432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.018400000408291817,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/spectral-initialization","display_name":"spectral initialization","score":0.7879999876022339},{"id":"https://openalex.org/keywords/weight-decay","display_name":"weight decay","score":0.8289999961853027},{"id":"https://openalex.org/keywords/batch-normalization","display_name":"batch normalization","score":0.9539999961853027},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"gradient descent","score":0.4129999876022339},{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.8100000023841858},{"id":"https://openalex.org/keywords/tensor-decomposition","display_name":"tensor decomposition","score":0.8289999961853027},{"id":"https://openalex.org/keywords/knowledge-distillation","display_name":"knowledge distillation","score":0.9860000014305115},{"id":"https://openalex.org/keywords/multi-head-self-attention","display_name":"multi-head self-attention","score":0.9449999928474426},{"id":"https://openalex.org/keywords/resnet","display_name":"ResNet","score":0.7129999995231628},{"id":"https://openalex.org/keywords/unsupervised-pretraining","display_name":"unsupervised pre-training","score":0.8489999771118164}],"concepts":[{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.8963354229927063},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7552280426025391},{"id":"https://openalex.org/C165464430","wikidata":"https://www.wikidata.org/wiki/Q1570441","display_name":"Parameterized complexity","level":2,"score":0.6042436361312866},{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.5618560314178467},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.529621422290802},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4919526278972626},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.47747159004211426},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.4619365930557251},{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.44582757353782654},{"id":"https://openalex.org/C42355184","wikidata":"https://www.wikidata.org/wiki/Q1361088","display_name":"Matrix decomposition","level":3,"score":0.4345756471157074},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.27186086773872375},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C158693339","wikidata":"https://www.wikidata.org/wiki/Q190524","display_name":"Eigenvalues and eigenvectors","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2105.01029","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2105.01029","pdf_url":"https://arxiv.org/pdf/2105.01029","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3158697128","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2105.01029","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2105.01029","doi":"https://doi.org/10.48550/arxiv.2105.01029","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2105.01029","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2105.01029","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2105.01029","pdf_url":"https://arxiv.org/pdf/2105.01029","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3133220799.pdf","grobid_xml":"https://content.openalex.org/works/W3133220799.grobid-xml"},"referenced_works_count":69,"referenced_works":["https://openalex.org/W4919037","https://openalex.org/W1492459858","https://openalex.org/W1503398984","https://openalex.org/W1677182931","https://openalex.org/W1686810756","https://openalex.org/W1836465849","https://openalex.org/W1981573335","https://openalex.org/W1993482030","https://openalex.org/W2000157792","https://openalex.org/W2108598243","https://openalex.org/W2114766824","https://openalex.org/W2144730813","https://openalex.org/W2161591461","https://openalex.org/W2173097616","https://openalex.org/W2178237821","https://openalex.org/W2179596243","https://openalex.org/W2181101938","https://openalex.org/W2194775991","https://openalex.org/W2295405790","https://openalex.org/W2613904329","https://openalex.org/W2626778328","https://openalex.org/W2738900641","https://openalex.org/W2741952635","https://openalex.org/W2768501777","https://openalex.org/W2786945063","https://openalex.org/W2805926765","https://openalex.org/W2808133870","https://openalex.org/W2892218381","https://openalex.org/W2894740066","https://openalex.org/W2898564876","https://openalex.org/W2904170036","https://openalex.org/W2907886210","https://openalex.org/W2908510526","https://openalex.org/W2913930213","https://openalex.org/W2945176031","https://openalex.org/W2945785363","https://openalex.org/W2947776571","https://openalex.org/W2947996606","https://openalex.org/W2963063862","https://openalex.org/W2963341956","https://openalex.org/W2963376662","https://openalex.org/W2963504252","https://openalex.org/W2963748441","https://openalex.org/W2963755427","https://openalex.org/W2963813662","https://openalex.org/W2964137095","https://openalex.org/W2964268168","https://openalex.org/W2969215180","https://openalex.org/W2970971581","https://openalex.org/W2971195897","https://openalex.org/W2981609437","https://openalex.org/W2981794819","https://openalex.org/W2995492258","https://openalex.org/W2995816250","https://openalex.org/W2996641029","https://openalex.org/W2997461673","https://openalex.org/W3005337910","https://openalex.org/W3010156474","https://openalex.org/W3014843068","https://openalex.org/W3035016149","https://openalex.org/W3035646933","https://openalex.org/W3041866211","https://openalex.org/W3087585563","https://openalex.org/W3088750390","https://openalex.org/W3108124733","https://openalex.org/W3118608800","https://openalex.org/W3209259379","https://openalex.org/W4285719527","https://openalex.org/W7213279897"],"related_works":["https://openalex.org/W4415800622","https://openalex.org/W4415798978","https://openalex.org/W4407186972","https://openalex.org/W3022529404","https://openalex.org/W2806265408","https://openalex.org/W2971553098","https://openalex.org/W2985509595","https://openalex.org/W7167892686","https://openalex.org/W7124161200","https://openalex.org/W4223571482"],"abstract_inverted_index":{"Factorized":[0],"layers--operations":[1],"parameterized":[2],"by":[3],"products":[4],"of":[5,13,23,75,87,107,116,125,149,155,162],"two":[6,43],"or":[7,188],"more":[8],"matrices--occur":[9],"in":[10],"a":[11,123,172,178,190],"variety":[12,124],"deep":[14,37],"learning":[15],"contexts,":[16],"including":[17],"compressed":[18],"model":[19,128,180],"training,":[20],"certain":[21],"types":[22],"knowledge":[24,167],"distillation,":[25,168],"and":[26,35,49,91,109,119,143,210],"multi-head":[27,201],"self-attention":[28],"architectures.":[29],"We":[30],"study":[31],"how":[32,88,196],"to":[33,60,73,103,137,200,204],"initialize":[34],"regularize":[36],"nets":[38],"containing":[39],"such":[40],"layers,":[41],"examining":[42],"simple,":[44,173],"understudied":[45],"schemes,":[46],"spectral":[47,117],"initialization":[48,90,118],"Frobenius":[50,120,169],"decay,":[51],"for":[52,64],"improving":[53],"their":[54,76],"performance.":[55],"The":[56],"guiding":[57],"insight":[58],"is":[59],"design":[61],"optimization":[62],"routines":[63],"these":[65],"networks":[66],"that":[67,74,132,176],"are":[68],"as":[69,71],"close":[70],"possible":[72],"well-tuned,":[77],"non-decomposed":[78],"counterparts;":[79],"we":[80,112,130,194],"back":[81],"this":[82],"intuition":[83],"with":[84,96,187],"an":[85],"analysis":[86],"the":[89,105,114,147,156,160],"regularization":[92],"schemes":[93,157,198],"impact":[94],"training":[95,150,183],"gradient":[97],"descent,":[98],"drawing":[99],"on":[100,146,207],"modern":[101],"attempts":[102],"understand":[104],"interplay":[106],"weight-decay":[108],"batch-normalization.":[110],"Empirically,":[111],"highlight":[113],"benefits":[115],"decay":[121,170],"across":[122],"settings.":[126],"In":[127],"compression,":[129],"show":[131,195],"they":[133],"enable":[134],"low-rank":[135],"methods":[136,145],"significantly":[138],"outperform":[139],"both":[140,197,208],"unstructured":[141],"sparsity":[142],"tensor":[144,163],"task":[148],"low-memory":[151],"residual":[152],"networks;":[153],"analogs":[154],"also":[158],"improve":[159],"performance":[161,206],"decomposition":[164],"techniques.":[165],"For":[166],"enables":[171],"overcomplete":[174],"baseline":[175],"yields":[177],"compact":[179],"from":[181],"over-parameterized":[182],"without":[184],"requiring":[185],"retraining":[186],"pruning":[189],"teacher":[191],"network.":[192],"Finally,":[193],"applied":[199],"attention":[202],"lead":[203],"improved":[205],"translation":[209],"unsupervised":[211],"pre-training.":[212]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3133220799","counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":14},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":1}],"updated_date":"2026-11-18T13:11:19.462570","created_date":"2021-03-01T00:00:00"}
