{"id":"https://openalex.org/W2804262948","doi":"https://doi.org/10.1109/tpami.2020.2990339","title":"Norm-Preservation: Why Residual Networks Can Become Extremely Deep?","display_name":"Norm-Preservation: Why Residual Networks Can Become Extremely Deep?","publication_year":2020,"publication_date":"2020-04-28","ids":{"openalex":"https://openalex.org/W2804262948","doi":"https://doi.org/10.1109/tpami.2020.2990339","mag":"2804262948","pmid":"https://pubmed.ncbi.nlm.nih.gov/32340937"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2020.2990339","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2020.2990339","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1805.07477","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5018280717","display_name":"Alireza Zaeemzadeh","orcid":"https://orcid.org/0000-0003-2980-1787"},"institutions":[{"id":"https://openalex.org/I106165777","display_name":"University of Central Florida","ror":"https://ror.org/036nfer12","country_code":"US","type":"education","lineage":["https://openalex.org/I106165777"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alireza Zaeemzadeh","raw_affiliation_strings":["School of Electrical Engineering and Computer Science, University of Central Florida, Orlando, FL, USA","School of Electrical Engineering and Computer Science, University of Central Florida, Orlando, FL, USA#TAB#"],"raw_orcid":"https://orcid.org/0000-0003-2980-1787","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering and Computer Science, University of Central Florida, Orlando, FL, USA","institution_ids":["https://openalex.org/I106165777"]},{"raw_affiliation_string":"School of Electrical Engineering and Computer Science, University of Central Florida, Orlando, FL, USA#TAB#","institution_ids":["https://openalex.org/I106165777"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109495751","display_name":"Nazanin Rahnavard","orcid":null},"institutions":[{"id":"https://openalex.org/I106165777","display_name":"University of Central Florida","ror":"https://ror.org/036nfer12","country_code":"US","type":"education","lineage":["https://openalex.org/I106165777"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nazanin Rahnavard","raw_affiliation_strings":["School of Electrical Engineering and Computer Science, University of Central Florida, Orlando, FL, USA","School of Electrical Engineering and Computer Science, University of Central Florida, Orlando, FL, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Electrical Engineering and Computer Science, University of Central Florida, Orlando, FL, USA","institution_ids":["https://openalex.org/I106165777"]},{"raw_affiliation_string":"School of Electrical Engineering and Computer Science, University of Central Florida, Orlando, FL, USA#TAB#","institution_ids":["https://openalex.org/I106165777"]}]},{"author_position":"last","author":{"id":null,"display_name":"Mubarak Shah","orcid":"https://orcid.org/0000-0002-8216-1128"},"institutions":[{"id":"https://openalex.org/I106165777","display_name":"University of Central Florida","ror":"https://ror.org/036nfer12","country_code":"US","type":"education","lineage":["https://openalex.org/I106165777"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mubarak Shah","raw_affiliation_strings":["School of Electrical Engineering and Computer Science, University of Central Florida, Orlando, FL, USA","School of Electrical Engineering and Computer Science, University of Central Florida, Orlando, FL, USA#TAB#"],"raw_orcid":"https://orcid.org/0000-0002-8216-1128","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering and Computer Science, University of Central Florida, Orlando, FL, USA","institution_ids":["https://openalex.org/I106165777"]},{"raw_affiliation_string":"School of Electrical Engineering and Computer Science, University of Central Florida, Orlando, FL, USA#TAB#","institution_ids":["https://openalex.org/I106165777"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.2187,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":{"value":0.83535522,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"43","issue":"11","first_page":"3980","last_page":"3990"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/residual-neural-network","display_name":"Residual neural network","score":0.7973241806030273},{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.7622393369674683},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7007652521133423},{"id":"https://openalex.org/keywords/norm","display_name":"Norm (philosophy)","score":0.6963143348693848},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.4390374422073364},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.4122805595397949},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.3973449766635895},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3791700601577759},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.37672188878059387},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2413783073425293}],"concepts":[{"id":"https://openalex.org/C2944601119","wikidata":"https://www.wikidata.org/wiki/Q43744058","display_name":"Residual neural network","level":3,"score":0.7973241806030273},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.7622393369674683},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7007652521133423},{"id":"https://openalex.org/C191795146","wikidata":"https://www.wikidata.org/wiki/Q3878446","display_name":"Norm (philosophy)","level":2,"score":0.6963143348693848},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4390374422073364},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4122805595397949},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3973449766635895},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3791700601577759},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.37672188878059387},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2413783073425293},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1109/tpami.2020.2990339","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2020.2990339","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:32340937","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/32340937","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null},{"id":"pmh:oai:arXiv.org:1805.07477","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1805.07477","pdf_url":"https://arxiv.org/pdf/1805.07477","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"mag:2804262948","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1805.07477.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1805.07477","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1805.07477","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1805.07477","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1805.07477","pdf_url":"https://arxiv.org/pdf/1805.07477","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[{"display_name":"Sustainable cities and communities","score":0.6200000047683716,"id":"https://metadata.un.org/sdg/11"}],"awards":[{"id":"https://openalex.org/G458936791","display_name":null,"funder_award_id":"1741431","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8490616318","display_name":null,"funder_award_id":"CCF-1718195","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320312530","display_name":"Office of the Director of National Intelligence","ror":"https://ror.org/01v3fsc55"},{"id":"https://openalex.org/F4320333051","display_name":"Intelligence Advanced Research Projects Activity","ror":"https://ror.org/01v3fsc55"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W194966468","https://openalex.org/W1533861849","https://openalex.org/W1677182931","https://openalex.org/W1826072219","https://openalex.org/W2045512849","https://openalex.org/W2073942547","https://openalex.org/W2137234026","https://openalex.org/W2161388792","https://openalex.org/W2194775991","https://openalex.org/W2302255633","https://openalex.org/W2565538933","https://openalex.org/W2623790698","https://openalex.org/W2758074773","https://openalex.org/W2766447205","https://openalex.org/W2797262859","https://openalex.org/W2797993462","https://openalex.org/W2949117887","https://openalex.org/W2950621961","https://openalex.org/W2952626150","https://openalex.org/W2953324412","https://openalex.org/W2962695743","https://openalex.org/W2963410064","https://openalex.org/W2963424732","https://openalex.org/W2963446085","https://openalex.org/W2963446712","https://openalex.org/W2963504252","https://openalex.org/W2963540976","https://openalex.org/W2963655672","https://openalex.org/W2963684275","https://openalex.org/W2963826371","https://openalex.org/W2964232029","https://openalex.org/W3118608800","https://openalex.org/W6626481562","https://openalex.org/W6631943919","https://openalex.org/W6638667902","https://openalex.org/W6683595889","https://openalex.org/W6713348437","https://openalex.org/W6714644935","https://openalex.org/W6729059855","https://openalex.org/W6731278065","https://openalex.org/W6734171162","https://openalex.org/W6738534199","https://openalex.org/W6738897456","https://openalex.org/W6741529945","https://openalex.org/W6744643100","https://openalex.org/W6750656786","https://openalex.org/W6751707404","https://openalex.org/W6752591435","https://openalex.org/W6755609140","https://openalex.org/W6787972765"],"related_works":["https://openalex.org/W2194775991","https://openalex.org/W2765720631","https://openalex.org/W2946641265","https://openalex.org/W2964911011","https://openalex.org/W2770522929","https://openalex.org/W3070904253","https://openalex.org/W2939745571","https://openalex.org/W2566079294","https://openalex.org/W2949978219","https://openalex.org/W2993726189","https://openalex.org/W2848033333","https://openalex.org/W3139471951","https://openalex.org/W2884368321","https://openalex.org/W3212331305","https://openalex.org/W71081281","https://openalex.org/W3004810722","https://openalex.org/W2055796545","https://openalex.org/W3098372854","https://openalex.org/W3083269515","https://openalex.org/W2962749646"],"abstract_inverted_index":{"Augmenting":[0],"neural":[1,56],"networks":[2,21,192],"with":[3,27],"skip":[4,40,53,62],"connections,":[5,41],"as":[6,92,180,186],"introduced":[7,174],"in":[8,55,64],"the":[9,14,18,37,49,61,65,70,73,98,101,131,135,140,151,155,173],"so-called":[10],"ResNet":[11,34,159],"architecture,":[12],"surprised":[13],"community":[15],"by":[16,35,110,124,163],"enabling":[17],"training":[19,190],"of":[20,22,39,51,72,100,134,158],"more":[23,93,167],"than":[24],"1,000":[25],"layers":[26,143],"significant":[28],"performance":[29,157],"gains.":[30],"This":[31],"paper":[32],"deciphers":[33],"analyzing":[36],"effect":[38],"and":[42,75,138,154,172,193],"puts":[43],"forward":[44],"new":[45,197],"theoretical":[46,106],"results":[47,171],"on":[48],"advantages":[50],"identity":[52],"connections":[54,63],"networks.":[57],"We":[58,86,120],"prove":[59],"that":[60,150],"residual":[66,94],"blocks":[67,95],"facilitate":[68],"preserving":[69],"norm":[71,168],"gradient,":[74],"lead":[76],"to":[77,129,179],"stable":[78],"back-propagation,":[79],"which":[80],"is":[81,103],"desirable":[82],"from":[83],"optimization":[84],"perspective.":[85],"also":[87,195],"show":[88],"that,":[89],"perhaps":[90],"surprisingly,":[91],"are":[96,108],"stacked,":[97],"norm-preservation":[99],"network":[102],"enhanced.":[104],"Our":[105,146,170],"arguments":[107],"supported":[109],"extensive":[111],"empirical":[112],"evidence.":[113],"Can":[114],"we":[115],"push":[116],"for":[117,176,189],"extra":[118,144],"norm-preservation?":[119],"answer":[121],"this":[122],"question":[123],"proposing":[125],"an":[126],"efficient":[127],"method":[128],"regularize":[130],"singular":[132],"values":[133],"convolution":[136],"operator":[137],"making":[139,164],"ResNet's":[141],"transition":[142],"norm-preserving.":[145],"numerical":[147],"investigations":[148],"demonstrate":[149],"learning":[152],"dynamics":[153],"classification":[156],"can":[160,183,194],"be":[161,184],"improved":[162],"it":[165],"even":[166],"preserving.":[169],"modification":[175],"ResNet,":[177],"referred":[178],"Procrustes":[181],"ResNets,":[182],"used":[185],"a":[187],"guide":[188],"deeper":[191,198],"inspire":[196],"architectures.":[199]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":6},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}