{"id":"https://openalex.org/W7080136626","doi":"https://doi.org/10.48550/arxiv.2509.03677","title":"Insights from Gradient Dynamics: Gradient Autoscaled Normalization","display_name":"Insights from Gradient Dynamics: Gradient Autoscaled Normalization","publication_year":2025,"publication_date":"2025-09-03","ids":{"openalex":"https://openalex.org/W7080136626","doi":"https://doi.org/10.48550/arxiv.2509.03677"},"language":"en","primary_location":{"id":"doi:10.48550/arxiv.2509.03677","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2509.03677","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2509.03677","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Yun, Vincent-Daniel","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Yun, Vincent-Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T12157","display_name":"Geochemistry and Geologic Mapping","score":0.6657999753952026,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12157","display_name":"Geochemistry and Geologic Mapping","score":0.6657999753952026,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13067","display_name":"Geological Modeling and Analysis","score":0.024900000542402267,"subfield":{"id":"https://openalex.org/subfields/1906","display_name":"Geochemistry and Petrology"},"field":{"id":"https://openalex.org/fields/19","display_name":"Earth and Planetary Sciences"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14311","display_name":"Electrical and Electromagnetic Research","score":0.01899999938905239,"subfield":{"id":"https://openalex.org/subfields/3107","display_name":"Atomic and Molecular Physics, and Optics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/normalization","display_name":"Normalization (sociology)","score":0.7452999949455261},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.5920000076293945},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.5151000022888184},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.447299987077713},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.44279998540878296},{"id":"https://openalex.org/keywords/gradient-method","display_name":"Gradient method","score":0.43880000710487366},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4131999909877777},{"id":"https://openalex.org/keywords/standard-deviation","display_name":"Standard deviation","score":0.40860000252723694}],"concepts":[{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.7452999949455261},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.5920000076293945},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.5151000022888184},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4821000099182129},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.447299987077713},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.44279998540878296},{"id":"https://openalex.org/C115680565","wikidata":"https://www.wikidata.org/wiki/Q5977448","display_name":"Gradient method","level":2,"score":0.43880000710487366},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4171999990940094},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4131999909877777},{"id":"https://openalex.org/C22679943","wikidata":"https://www.wikidata.org/wiki/Q159375","display_name":"Standard deviation","level":2,"score":0.40860000252723694},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.3772999942302704},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.35409998893737793},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.34369999170303345},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.32249999046325684},{"id":"https://openalex.org/C19444555","wikidata":"https://www.wikidata.org/wiki/Q212750","display_name":"Flattening","level":2,"score":0.30570000410079956},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.3046000003814697},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.29840001463890076},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.29429998993873596},{"id":"https://openalex.org/C177918212","wikidata":"https://www.wikidata.org/wiki/Q803623","display_name":"Perturbation (astronomy)","level":2,"score":0.26989999413490295},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.26820001006126404},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.2678000032901764}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2509.03677","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2509.03677","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2509.03677","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2509.03677","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Gradient":[0],"dynamics":[1],"play":[2],"a":[3,54],"central":[4],"role":[5],"in":[6,45],"determining":[7],"the":[8,42,80,108,118],"stability":[9],"and":[10,27,40,74,87,123,126],"generalization":[11],"of":[12,24,30,110],"deep":[13],"neural":[14],"networks.":[15,47],"In":[16],"this":[17],"work,":[18],"we":[19,52],"provide":[20,128],"an":[21],"empirical":[22,124],"analysis":[23],"how":[25],"variance":[26],"standard":[28],"deviation":[29],"gradients":[31],"evolve":[32],"during":[33],"training,":[34],"showing":[35],"consistent":[36],"changes":[37],"across":[38],"layers":[39],"at":[41],"global":[43],"scale":[44],"convolutional":[46],"Motivated":[48],"by":[49],"these":[50],"observations,":[51],"propose":[53],"hyperparameter-free":[55],"gradient":[56,61,113],"normalization":[57],"method":[58,92],"that":[59,90],"aligns":[60],"scaling":[62],"with":[63,84],"their":[64],"natural":[65],"evolution.":[66],"This":[67],"approach":[68],"prevents":[69],"unintended":[70],"amplification,":[71],"stabilizes":[72],"optimization,":[73],"preserves":[75],"convergence":[76],"guarantees.":[77],"Experiments":[78],"on":[79],"challenging":[81],"CIFAR-100":[82],"benchmark":[83],"ResNet-20,":[85],"ResNet-56,":[86],"VGG-16-BN":[88],"demonstrate":[89],"our":[91,105],"maintains":[93],"or":[94],"improves":[95],"test":[96],"accuracy":[97],"even":[98],"under":[99],"strong":[100],"generalization.":[101],"Beyond":[102],"practical":[103],"performance,":[104],"study":[106],"highlights":[107],"importance":[109],"directly":[111],"tracking":[112],"dynamics,":[114],"aiming":[115],"to":[116,127],"bridge":[117],"gap":[119],"between":[120],"theoretical":[121],"expectations":[122],"behaviors,":[125],"insights":[129],"for":[130],"future":[131],"optimization":[132],"research.":[133]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-10T00:00:00"}