{"id":"https://openalex.org/W3089807438","doi":"https://doi.org/10.3233/faia200253","title":"Layer-Wise Adaptive Gradient Sparsification for Distributed Deep Learning with Convergence Guarantees","display_name":"Layer-Wise Adaptive Gradient Sparsification for Distributed Deep Learning with Convergence Guarantees","publication_year":2020,"publication_date":"2020-01-01","ids":{"openalex":"https://openalex.org/W3089807438","doi":"https://doi.org/10.3233/faia200253","mag":"3089807438"},"language":"en","primary_location":{"id":"doi:10.3233/faia200253","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia200253","pdf_url":null,"source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"},"type":"book-chapter","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.3233/faia200253","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016836702","display_name":"Shaohuai Shi","orcid":"https://orcid.org/0000-0002-1418-5160"},"institutions":[{"id":"https://openalex.org/I141568987","display_name":"Hong Kong Baptist University","ror":"https://ror.org/0145fw131","country_code":"HK","type":"education","lineage":["https://openalex.org/I141568987"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shi Shaohuai","raw_affiliation_strings":["Hong Kong Baptist University, Hong Kong, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hong Kong Baptist University, Hong Kong, China","institution_ids":["https://openalex.org/I141568987"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048313055","display_name":"Zhenheng Tang","orcid":"https://orcid.org/0000-0001-8769-9974"},"institutions":[{"id":"https://openalex.org/I141568987","display_name":"Hong Kong Baptist University","ror":"https://ror.org/0145fw131","country_code":"HK","type":"education","lineage":["https://openalex.org/I141568987"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Tang Zhenheng","raw_affiliation_strings":["Hong Kong Baptist University, Hong Kong, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hong Kong Baptist University, Hong Kong, China","institution_ids":["https://openalex.org/I141568987"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Wang Qiang","orcid":null},"institutions":[{"id":"https://openalex.org/I141568987","display_name":"Hong Kong Baptist University","ror":"https://ror.org/0145fw131","country_code":"HK","type":"education","lineage":["https://openalex.org/I141568987"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Wang Qiang","raw_affiliation_strings":["Hong Kong Baptist University, Hong Kong, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hong Kong Baptist University, Hong Kong, China","institution_ids":["https://openalex.org/I141568987"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100675370","display_name":"Kaiyong Zhao","orcid":null},"institutions":[{"id":"https://openalex.org/I141568987","display_name":"Hong Kong Baptist University","ror":"https://ror.org/0145fw131","country_code":"HK","type":"education","lineage":["https://openalex.org/I141568987"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhao Kaiyong","raw_affiliation_strings":["Hong Kong Baptist University, Hong Kong, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hong Kong Baptist University, Hong Kong, China","institution_ids":["https://openalex.org/I141568987"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100730785","display_name":"Xiaowen Chu","orcid":"https://orcid.org/0000-0001-9745-4372"},"institutions":[{"id":"https://openalex.org/I141568987","display_name":"Hong Kong Baptist University","ror":"https://ror.org/0145fw131","country_code":"HK","type":"education","lineage":["https://openalex.org/I141568987"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Chu Xiaowen","raw_affiliation_strings":["Hong Kong Baptist University, Hong Kong, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hong Kong Baptist University, Hong Kong, China","institution_ids":["https://openalex.org/I141568987"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I141568987"],"apc_list":null,"apc_paid":null,"fwci":1.5585,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.86761798,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1467","last_page":"1474"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10500","display_name":"Sparse and Compressive Sensing Techniques","score":0.9902999997138977,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9842000007629395,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.7061659097671509},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.562436580657959},{"id":"https://openalex.org/keywords/distributed-learning","display_name":"Distributed learning","score":0.5310887098312378},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4770331084728241},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4237239360809326},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.11941224336624146},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.0664030909538269}],"concepts":[{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.7061659097671509},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.562436580657959},{"id":"https://openalex.org/C2779582901","wikidata":"https://www.wikidata.org/wiki/Q21013010","display_name":"Distributed learning","level":2,"score":0.5310887098312378},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4770331084728241},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4237239360809326},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.11941224336624146},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0664030909538269},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.3233/faia200253","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia200253","pdf_url":null,"source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-105136","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-105136","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference paper"},{"id":"mag:3089807438","is_oa":false,"landing_page_url":"https://dblp.uni-trier.de/db/journals/corr/corr1911.html#abs-1911-08727","pdf_url":null,"source":{"id":"https://openalex.org/S4306418308","display_name":"European Conference on Artificial Intelligence","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":"European Conference on Artificial Intelligence","raw_type":null}],"best_oa_location":{"id":"doi:10.3233/faia200253","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia200253","pdf_url":null,"source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2731899572","https://openalex.org/W3215138031","https://openalex.org/W3009238340","https://openalex.org/W4360585206","https://openalex.org/W4321369474","https://openalex.org/W4285208911","https://openalex.org/W3082895349","https://openalex.org/W4213079790","https://openalex.org/W2248239756","https://openalex.org/W3086377361"],"abstract_inverted_index":{"To":[0],"reduce":[1],"the":[2,28,37,53,59,66,69,74,122,134,142,152,165,185,189,204],"long":[3],"training":[4],"time":[5],"of":[6,25,55,109,125,131,136,145,168,192],"large":[7],"deep":[8],"neural":[9],"network":[10],"(DNN)":[11],"models,":[12],"distributed":[13,84],"synchronous":[14],"stochastic":[15],"gradient":[16,45,60,97],"descent":[17],"(S-SGD)":[18],"is":[19,34],"commonly":[20],"used":[21],"on":[22,196],"a":[23,82,93,106,175,197],"cluster":[24,199],"workers.":[26],"However,":[27],"speedup":[29],"brought":[30],"by":[31,36],"multiple":[32],"workers":[33],"limited":[35],"communication":[38,56,67,153],"overhead.":[39],"Two":[40],"approaches,":[41],"namely":[42],"pipelining":[43,75],"and":[44,71,162,188,207],"sparsification,":[46],"have":[47],"been":[48],"separately":[49],"proposed":[50],"to":[51,121,150,183],"alleviate":[52],"impact":[54],"overheads.":[57],"Yet,":[58],"sparsification":[61,98],"methods":[62],"can":[63,118],"only":[64],"initiate":[65],"after":[68],"backpropagation,":[70],"hence":[72],"miss":[73],"opportunity.":[76],"In":[77,101],"this":[78],"paper,":[79],"we":[80],"propose":[81],"new":[83],"optimization":[85],"method":[86],"named":[87],"LAGS-SGD,":[88,102],"which":[89],"combines":[90],"S-SGD":[91,173,206,210],"with":[92,139],"novel":[94],"layer-wise":[95,129],"adaptive":[96,120,143],"(LAGS)":[99],"scheme.":[100],"every":[103],"worker":[104],"selects":[105],"small":[107],"set":[108],"'significant'":[110],"gradients":[111],"from":[112],"each":[113],"layer":[114],"independently":[115],"whose":[116],"size":[117],"be":[119],"communication-to-computation":[123],"ratio":[124],"that":[126,157,201],"layer.":[127],"The":[128],"nature":[130,144],"LAGS-SGD":[132,146,158,202],"opens":[133],"opportunity":[135],"overlapping":[137],"communications":[138],"computations,":[140],"while":[141],"makes":[147],"it":[148,163],"flexible":[149],"control":[151],"time.":[154],"We":[155],"prove":[156],"has":[159,164],"convergence":[160,169,190],"guarantees":[161],"same":[166],"order":[167],"rate":[170],"as":[171],"vanilla":[172],"under":[174],"weak":[176],"analytical":[177,186],"assumption.":[178],"Extensive":[179],"experiments":[180],"are":[181],"conducted":[182],"verify":[184],"assumption":[187],"performance":[191],"LAGS-SGD.":[193],"Experimental":[194],"results":[195],"16-GPU":[198],"show":[200],"outperforms":[203],"original":[205],"existing":[208],"sparsified":[209],"without":[211],"losing":[212],"obvious":[213],"model":[214],"accuracy.":[215]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2020,"cited_by_count":2}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}