{"id":"https://openalex.org/W2969534524","doi":"https://doi.org/10.1109/access.2019.2937139","title":"Adaptive Weight Decay for Deep Neural Networks","display_name":"Adaptive Weight Decay for Deep Neural Networks","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2969534524","doi":"https://doi.org/10.1109/access.2019.2937139","mag":"2969534524"},"language":"en","primary_location":{"id":"doi:10.1109/access.2019.2937139","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2019.2937139","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08811458.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08811458.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102774458","display_name":"Kensuke Nakamura","orcid":"https://orcid.org/0000-0002-6858-3551"},"institutions":[{"id":"https://openalex.org/I67900169","display_name":"Chung-Ang University","ror":"https://ror.org/01r024a98","country_code":"KR","type":"education","lineage":["https://openalex.org/I67900169"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Kensuke Nakamura","raw_affiliation_strings":["Computer Science Department, Chung-Ang University, Seoul, South Korea"],"raw_orcid":"https://orcid.org/0000-0002-6858-3551","affiliations":[{"raw_affiliation_string":"Computer Science Department, Chung-Ang University, Seoul, South Korea","institution_ids":["https://openalex.org/I67900169"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5065943331","display_name":"Byung\u2010Woo Hong","orcid":"https://orcid.org/0000-0003-2752-3939"},"institutions":[{"id":"https://openalex.org/I67900169","display_name":"Chung-Ang University","ror":"https://ror.org/01r024a98","country_code":"KR","type":"education","lineage":["https://openalex.org/I67900169"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Byung-Woo Hong","raw_affiliation_strings":["Computer Science Department, Chung-Ang University, Seoul, South Korea"],"raw_orcid":"https://orcid.org/0000-0003-2752-3939","affiliations":[{"raw_affiliation_string":"Computer Science Department, Chung-Ang University, Seoul, South Korea","institution_ids":["https://openalex.org/I67900169"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I67900169"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":2.047,"has_fulltext":false,"cited_by_count":57,"citation_normalized_percentile":{"value":0.87980576,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"7","issue":null,"first_page":"118857","last_page":"118865"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mnist-database","display_name":"MNIST database","score":0.8111905455589294},{"id":"https://openalex.org/keywords/regularization","display_name":"Regularization (linguistics)","score":0.7013144493103027},{"id":"https://openalex.org/keywords/sigmoid-function","display_name":"Sigmoid function","score":0.6881909966468811},{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.5950621962547302},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5420395731925964},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5150461196899414},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.46976467967033386},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4564499855041504},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.4443740248680115},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.435674786567688},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.4276397228240967},{"id":"https://openalex.org/keywords/applied-mathematics","display_name":"Applied mathematics","score":0.42597201466560364},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.38726457953453064},{"id":"https://openalex.org/keywords/mathematical-analysis","display_name":"Mathematical analysis","score":0.11685618758201599}],"concepts":[{"id":"https://openalex.org/C190502265","wikidata":"https://www.wikidata.org/wiki/Q17069496","display_name":"MNIST database","level":3,"score":0.8111905455589294},{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.7013144493103027},{"id":"https://openalex.org/C81388566","wikidata":"https://www.wikidata.org/wiki/Q526668","display_name":"Sigmoid function","level":3,"score":0.6881909966468811},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.5950621962547302},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5420395731925964},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5150461196899414},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46976467967033386},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4564499855041504},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.4443740248680115},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.435674786567688},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.4276397228240967},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.42597201466560364},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.38726457953453064},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.11685618758201599},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2019.2937139","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2019.2937139","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08811458.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:06de983ac8854da5ac03572e37dd7d62","is_oa":true,"landing_page_url":"https://doaj.org/article/06de983ac8854da5ac03572e37dd7d62","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 7, Pp 118857-118865 (2019)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2019.2937139","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2019.2937139","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08811458.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1407564792","display_name":null,"funder_award_id":"NRF-2017R1A2B4006023","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"},{"id":"https://openalex.org/G6487955083","display_name":null,"funder_award_id":"NRF-2018R1A4A1059731","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"}],"funders":[{"id":"https://openalex.org/F4320322120","display_name":"National Research Foundation of Korea","ror":"https://ror.org/013aysd81"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":101,"referenced_works":["https://openalex.org/W114517082","https://openalex.org/W1498436455","https://openalex.org/W1522301498","https://openalex.org/W1582774210","https://openalex.org/W1686810756","https://openalex.org/W1798702550","https://openalex.org/W1806891645","https://openalex.org/W1826234144","https://openalex.org/W1836465849","https://openalex.org/W1994616650","https://openalex.org/W2033137841","https://openalex.org/W2091825929","https://openalex.org/W2095705004","https://openalex.org/W2097117768","https://openalex.org/W2105875671","https://openalex.org/W2107438106","https://openalex.org/W2112796928","https://openalex.org/W2117229703","https://openalex.org/W2136836265","https://openalex.org/W2144513243","https://openalex.org/W2146502635","https://openalex.org/W2146917784","https://openalex.org/W2194775991","https://openalex.org/W2301983558","https://openalex.org/W2302255633","https://openalex.org/W2331143823","https://openalex.org/W2531409750","https://openalex.org/W2574426558","https://openalex.org/W2591954064","https://openalex.org/W2605372163","https://openalex.org/W2741107118","https://openalex.org/W2750384547","https://openalex.org/W2752159661","https://openalex.org/W2777891128","https://openalex.org/W2788597178","https://openalex.org/W2803300317","https://openalex.org/W2803974531","https://openalex.org/W2804386825","https://openalex.org/W2884132298","https://openalex.org/W2912811302","https://openalex.org/W2950277768","https://openalex.org/W2950928354","https://openalex.org/W2951595529","https://openalex.org/W2951841718","https://openalex.org/W2952626150","https://openalex.org/W2962795635","https://openalex.org/W2962851402","https://openalex.org/W2962947632","https://openalex.org/W2963139509","https://openalex.org/W2963243330","https://openalex.org/W2963334011","https://openalex.org/W2963433607","https://openalex.org/W2963446712","https://openalex.org/W2963505370","https://openalex.org/W2963702144","https://openalex.org/W2963734024","https://openalex.org/W2964121744","https://openalex.org/W2964241627","https://openalex.org/W2969945254","https://openalex.org/W3093329015","https://openalex.org/W3118608800","https://openalex.org/W3137695714","https://openalex.org/W3147765605","https://openalex.org/W4235765578","https://openalex.org/W4254751698","https://openalex.org/W4294622503","https://openalex.org/W4299971819","https://openalex.org/W4300197707","https://openalex.org/W6629815555","https://openalex.org/W6631190155","https://openalex.org/W6637373629","https://openalex.org/W6638209102","https://openalex.org/W6638667902","https://openalex.org/W6638836233","https://openalex.org/W6674330103","https://openalex.org/W6676105031","https://openalex.org/W6677975206","https://openalex.org/W6679955943","https://openalex.org/W6681151457","https://openalex.org/W6681435938","https://openalex.org/W6693797521","https://openalex.org/W6698183232","https://openalex.org/W6729595453","https://openalex.org/W6732021836","https://openalex.org/W6734171162","https://openalex.org/W6736583452","https://openalex.org/W6742248563","https://openalex.org/W6743688258","https://openalex.org/W6744265424","https://openalex.org/W6747543534","https://openalex.org/W6748324676","https://openalex.org/W6748543269","https://openalex.org/W6751777967","https://openalex.org/W6751888367","https://openalex.org/W6751983209","https://openalex.org/W6753411400","https://openalex.org/W6763485134","https://openalex.org/W6766357795","https://openalex.org/W6779882468","https://openalex.org/W6787972765","https://openalex.org/W6793127723"],"related_works":["https://openalex.org/W3082263874","https://openalex.org/W3004759583","https://openalex.org/W4391093647","https://openalex.org/W3173926637","https://openalex.org/W4385524141","https://openalex.org/W4297776111","https://openalex.org/W3018979822","https://openalex.org/W3026616975","https://openalex.org/W2989784533","https://openalex.org/W4288018014"],"abstract_inverted_index":{"Regularization":[0],"in":[1,41,60,95,104,131,150],"the":[2,23,37,42,49,57,61,68,73,82,88,96,99,113,122,134,140,146,153,193],"optimization":[3,155],"of":[4,19,22,44,63,76,91,98,124,136,148,178],"deep":[5],"neural":[6,167],"networks":[7],"is":[8,28,52,129],"often":[9],"critical":[10],"to":[11,16,29,55,67,80,133,152,173,188],"avoid":[12],"undesirable":[13],"over-fitting":[14],"leading":[15,187],"better":[17,189],"generalization":[18,186],"model.":[20],"One":[21],"most":[24],"popular":[25,158],"regularization":[26,125],"algorithms":[27,156],"impose":[30],"L":[31],"<sub":[32],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[33],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">2</sub>":[34],"penalty":[35],"on":[36,72],"model":[38,58,92],"parameters":[39,59],"resulting":[40],"decay":[43,50],"parameters,":[45],"called":[46,108],"weight-decay,":[47,77],"and":[48,93,121,163,195],"rate":[51,75],"generally":[53],"constant":[54,74],"all":[56,192],"course":[62],"optimization.":[64],"In":[65],"contrast":[66],"previous":[69],"approach":[70],"based":[71],"we":[78],"propose":[79],"consider":[81],"residual":[83],"that":[84,183],"measures":[85],"dissimilarity":[86],"between":[87],"current":[89],"state":[90],"observations":[94],"determination":[97],"weight-decay":[100,110],"for":[101,126],"each":[102,119,127],"parameter":[103,128],"an":[105],"adaptive":[106,109],"way,":[107],"(AdaDecay)":[111],"where":[112],"gradient":[114,138],"norms":[115],"are":[116],"normalized":[117],"within":[118],"layer":[120],"degree":[123],"determined":[130],"proportional":[132],"magnitude":[135],"its":[137],"using":[139,157],"sigmoid":[141],"function.":[142],"We":[143],"empirically":[144],"demonstrate":[145],"effectiveness":[147],"AdaDecay":[149,184],"comparison":[151],"state-of-the-art":[154],"benchmark":[159],"datasets:":[160],"MNIST,":[161],"Fashion-MNIST,":[162],"CIFAR-10":[164],"with":[165],"conventional":[166],"network":[168],"models":[169],"ranging":[170],"from":[171],"shallow":[172],"deep.":[174],"The":[175],"quantitative":[176],"evaluation":[177],"our":[179],"proposed":[180],"algorithm":[181],"indicates":[182],"improves":[185],"accuracy":[190],"across":[191],"datasets":[194],"models.":[196]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":13},{"year":2023,"cited_by_count":11},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":7},{"year":2020,"cited_by_count":4}],"updated_date":"2026-07-17T09:13:05.818461","created_date":"2025-10-10T00:00:00"}