{"id":"https://openalex.org/W3167550568","doi":"https://doi.org/10.48550/arxiv.2106.09475","title":"Backward Gradient Normalization in Deep Neural Networks","display_name":"Backward Gradient Normalization in Deep Neural Networks","publication_year":2021,"publication_date":"2021-06-17","ids":{"openalex":"https://openalex.org/W3167550568","doi":"https://doi.org/10.48550/arxiv.2106.09475","mag":"3167550568"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2106.09475","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2106.09475","pdf_url":"https://arxiv.org/pdf/2106.09475","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2106.09475","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031354297","display_name":"Alejandro Sanchez Cabana","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I63634437","display_name":"Universidad Aut\u00f3noma de Madrid","ror":"https://ror.org/01cby8j38","country_code":"ES","type":"education","lineage":["https://openalex.org/I63634437"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Cabana, Alejandro","raw_affiliation_strings":["[Autonomous University of Madrid]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"[Autonomous University of Madrid]","institution_ids":["https://openalex.org/I63634437"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034298577","display_name":"Luis F. Lago-Fern\u00e1ndez","orcid":"https://orcid.org/0000-0001-8639-8731","observed_orcids":["https://orcid.org/0000-0001-8639-8731"]},"institutions":[{"id":"https://openalex.org/I63634437","display_name":"Universidad Aut\u00f3noma de Madrid","ror":"https://ror.org/01cby8j38","country_code":"ES","type":"education","lineage":["https://openalex.org/I63634437"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Lago-Fern\u00e1ndez, Luis F.","raw_affiliation_strings":["[Autonomous University of Madrid]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"[Autonomous University of Madrid]","institution_ids":["https://openalex.org/I63634437"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I63634437"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.8241999745368958,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.8241999745368958,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.07559999823570251,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.05260000005364418,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/gradient-normalization","display_name":"gradient normalization","score":0.9589999914169312},{"id":"https://openalex.org/keywords/backward-pass","display_name":"backward pass","score":0.7570000290870667},{"id":"https://openalex.org/keywords/normalization-layers","display_name":"normalization layers","score":0.8960000276565552},{"id":"https://openalex.org/keywords/vanishing-gradient","display_name":"gradient vanishing","score":0.8759999871253967},{"id":"https://openalex.org/keywords/exploding-gradients","display_name":"gradient explosion","score":0.9399999976158142},{"id":"https://openalex.org/keywords/gradient-flow","display_name":"gradient flow","score":0.5989999771118164},{"id":"https://openalex.org/keywords/deep-neural-network","display_name":"deep neural network","score":0.8650000095367432},{"id":"https://openalex.org/keywords/weight-update","display_name":"weight updating","score":0.421999990940094},{"id":"https://openalex.org/keywords/training-stability","display_name":"training stability","score":0.4569999873638153}],"concepts":[{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.804170548915863},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.5755817294120789},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5366880893707275},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5155110359191895},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5124507546424866},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.06285718083381653},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2106.09475","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2106.09475","pdf_url":"https://arxiv.org/pdf/2106.09475","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2106.09475","doi":"https://doi.org/10.48550/arxiv.2106.09475","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2106.09475","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2106.09475","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2106.09475","pdf_url":"https://arxiv.org/pdf/2106.09475","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W836608889","https://openalex.org/W1665214252","https://openalex.org/W1806891645","https://openalex.org/W1815076433","https://openalex.org/W2064675550","https://openalex.org/W2069143585","https://openalex.org/W2076063813","https://openalex.org/W2107878631","https://openalex.org/W2156387975","https://openalex.org/W2194775991","https://openalex.org/W2561193401","https://openalex.org/W2737818580","https://openalex.org/W2785921341","https://openalex.org/W2919115771","https://openalex.org/W2949117887","https://openalex.org/W2963685250","https://openalex.org/W2964121744"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2591697403","https://openalex.org/W2953716828","https://openalex.org/W2904857019","https://openalex.org/W2944728705","https://openalex.org/W3011538607","https://openalex.org/W4287755480","https://openalex.org/W3113607506","https://openalex.org/W2898211994"],"abstract_inverted_index":{"We":[0],"introduce":[1],"a":[2,46],"new":[3,72],"technique":[4,73],"for":[5],"gradient":[6,48,81],"normalization":[7,21,32],"during":[8,16],"neural":[9,67],"network":[10,29,54,94],"training.":[11],"The":[12],"gradients":[13],"are":[14],"rescaled":[15],"the":[17,28,52,71,80,84,89],"backward":[18],"pass":[19],"using":[20],"layers":[22,55,91],"introduced":[23],"at":[24],"certain":[25],"points":[26],"within":[27],"architecture.":[30],"These":[31],"nodes":[33],"do":[34,75],"not":[35],"affect":[36],"forward":[37],"activity":[38],"propagation,":[39],"but":[40],"modify":[41],"backpropagation":[42],"equations":[43],"to":[44],"permit":[45],"well-scaled":[47],"flow":[49],"that":[50,70],"reaches":[51],"deepest":[53,90],"without":[56],"experimenting":[57],"vanishing":[58],"or":[59],"explosion.":[60],"Results":[61],"on":[62,96],"tests":[63],"with":[64],"very":[65],"deep":[66],"networks":[68],"show":[69],"can":[74],"an":[76],"effective":[77],"control":[78],"of":[79,86],"norm,":[82],"allowing":[83],"update":[85],"weights":[87],"in":[88],"and":[92],"improving":[93],"accuracy":[95],"several":[97],"experimental":[98],"conditions.":[99]},"counts_by_year":[],"updated_date":"2026-09-29T14:09:16.559479","created_date":"2025-10-10T00:00:00"}