{"id":"https://openalex.org/W4224325899","doi":"https://doi.org/10.1145/3485447.3511989","title":"Not All Layers Are Equal: A Layer-Wise Adaptive Approach Toward Large-Scale DNN Training","display_name":"Not All Layers Are Equal: A Layer-Wise Adaptive Approach Toward Large-Scale DNN Training","publication_year":2022,"publication_date":"2022-04-25","ids":{"openalex":"https://openalex.org/W4224325899","doi":"https://doi.org/10.1145/3485447.3511989"},"language":"en","primary_location":{"id":"doi:10.1145/3485447.3511989","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3485447.3511989","pdf_url":null,"source":{"id":"https://openalex.org/S4363608783","display_name":"Proceedings of the ACM Web Conference 2022","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Web Conference 2022","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042828356","display_name":"Yunyong Ko","orcid":"https://orcid.org/0000-0003-1283-4697"},"institutions":[{"id":"https://openalex.org/I4575257","display_name":"Hanyang University","ror":"https://ror.org/046865y68","country_code":"KR","type":"education","lineage":["https://openalex.org/I4575257"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Yunyong Ko","raw_affiliation_strings":["Hanyang University, Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hanyang University, Republic of Korea","institution_ids":["https://openalex.org/I4575257"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100405086","display_name":"Dongwon Lee","orcid":"https://orcid.org/0000-0001-8371-7629"},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dongwon Lee","raw_affiliation_strings":["The Pennsylvania State University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Pennsylvania State University, USA","institution_ids":["https://openalex.org/I130769515"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100656150","display_name":"Sang\u2010Wook Kim","orcid":"https://orcid.org/0000-0002-6345-9084"},"institutions":[{"id":"https://openalex.org/I4575257","display_name":"Hanyang University","ror":"https://ror.org/046865y68","country_code":"KR","type":"education","lineage":["https://openalex.org/I4575257"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Sang-Wook Kim","raw_affiliation_strings":["Hanyang University, Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hanyang University, Republic of Korea","institution_ids":["https://openalex.org/I4575257"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1851","last_page":"1859"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9882000088691711,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9882000088691711,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12702","display_name":"Brain Tumor Detection and Classification","score":0.9857000112533569,"subfield":{"id":"https://openalex.org/subfields/2808","display_name":"Neurology"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9768999814987183,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.7689322233200073},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6890058517456055},{"id":"https://openalex.org/keywords/layer","display_name":"Layer (electronics)","score":0.6467335224151611},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.634422779083252},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.48471879959106445},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4133053421974182},{"id":"https://openalex.org/keywords/materials-science","display_name":"Materials science","score":0.08320656418800354}],"concepts":[{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.7689322233200073},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6890058517456055},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.6467335224151611},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.634422779083252},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48471879959106445},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4133053421974182},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.08320656418800354},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3485447.3511989","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3485447.3511989","pdf_url":null,"source":{"id":"https://openalex.org/S4363608783","display_name":"Proceedings of the ACM Web Conference 2022","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Web Conference 2022","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1726689462","display_name":"Developing and Evaluating Fraud Informatics Curriculum among Institutions in the Appalachian Region","funder_award_id":"1820609","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6346334673","display_name":null,"funder_award_id":"2020R1A2B5B03001960","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320322120","display_name":"National Research Foundation of Korea","ror":"https://ror.org/013aysd81"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W2108598243","https://openalex.org/W2117539524","https://openalex.org/W2119391823","https://openalex.org/W2194775991","https://openalex.org/W2892283076","https://openalex.org/W2912917402","https://openalex.org/W2914500297","https://openalex.org/W2988070836","https://openalex.org/W3023944502","https://openalex.org/W3082665562","https://openalex.org/W3137236623","https://openalex.org/W3156044630","https://openalex.org/W3173185171","https://openalex.org/W3175786688","https://openalex.org/W3208284577","https://openalex.org/W6684859321"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W3107602296","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347"],"abstract_inverted_index":{"A":[0],"large-batch":[1,71,230],"training":[2,72,172,228],"with":[3,73,184],"data":[4],"parallelism":[5],"is":[6],"a":[7,14,85,89,133,148,162],"widely":[8],"adopted":[9],"approach":[10,135],"to":[11,58,153,166,212],"efficiently":[12],"train":[13],"large":[15],"deep":[16],"neural":[17],"network":[18],"(DNN)":[19],"model.":[20],"Large-batch":[21],"training,":[22,117,145],"however,":[23,67],"often":[24,79],"suffers":[25],"from":[26],"the":[27,30,55,81,109,121,124,138,168,171,193,197,204,221,233],"problem":[28],"of":[29,35,123,170,186,199,235],"model":[31,82,126],"quality":[32,83],"degradation":[33,122],"because":[34],"its":[36,178],"fewer":[37,214],"iterations.":[38],"To":[39,98],"alleviate":[40],"this":[41,99,129],"problem,":[42],"in":[43,116,120],"general,":[44],"learning":[45,56,139],"rate":[46,57,140],"(lr)":[47],"scaling":[48,76,106,141,152],"methods":[49,77,95,107],"have":[50],"been":[51],"applied,":[52],"which":[53,118],"increases":[54],"make":[59],"an":[60],"update":[61],"larger":[62],"at":[63],"each":[64,157,174],"iteration.":[65],"Unfortunately,":[66],"we":[68,101,131],"observe":[69],"that":[70,103,190],"state-of-the-art":[74,223],"lr":[75,94,105,151,155],"still":[78],"degrade":[80],"when":[84],"batch":[86,187,209],"size":[87],"crosses":[88],"specific":[90],"limit,":[91],"rendering":[92],"such":[93],"less":[96],"useful.":[97],"phenomenon,":[100],"hypothesize":[102],"existing":[104,222],"overlook":[108],"subtle":[110],"but":[111],"important":[112],"differences":[113],"across":[114,207],"\u201clayers\u201d":[115],"results":[119],"overall":[125],"quality.":[127],"From":[128],"hypothesis,":[130],"propose":[132],"novel":[134],"(LENA)":[136],"toward":[137],"for":[142,156,173,227],"large-scale":[143],"DNN":[144],"employing:":[146],"(1)":[147,202],"layer-wise":[149,163],"adaptive":[150],"adjust":[154],"layer":[158,175],"individually,":[159],"and":[160,176,216,225],"(2)":[161,226],"state-aware":[164],"warm-up":[165,179],"track":[167],"state":[169],"finish":[177],"automatically.":[180],"The":[181],"comprehensive":[182],"evaluation":[183],"variations":[185],"sizes":[188,210],"demonstrates":[189],"LENA":[191],"achieves":[192],"target":[194],"accuracy":[195,198],"(i.e.,":[196],"single-worker":[200],"training):":[201],"within":[203],"fewest":[205],"iterations":[206,215],"different":[208],"(up":[211],"45.2%":[213],"44.7%":[217],"shorter":[218],"time":[219],"than":[220],"method),":[224],"very":[229],"sizes,":[231],"surpassing":[232],"limits":[234],"all":[236],"baselines.":[237]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}