{"id":"https://openalex.org/W3047696039","doi":"https://doi.org/10.1145/3404397.3404401","title":"Dual-Way Gradient Sparsification for Asynchronous Distributed Deep Learning","display_name":"Dual-Way Gradient Sparsification for Asynchronous Distributed Deep Learning","publication_year":2020,"publication_date":"2020-08-09","ids":{"openalex":"https://openalex.org/W3047696039","doi":"https://doi.org/10.1145/3404397.3404401","mag":"3047696039"},"language":"en","primary_location":{"id":"doi:10.1145/3404397.3404401","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404397.3404401","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"49th International Conference on Parallel Processing - ICPP","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091674821","display_name":"Zijie Yan","orcid":null},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zijie Yan","raw_affiliation_strings":["Sun Yat-sen University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sun Yat-sen University","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041031990","display_name":"Danyang Xiao","orcid":"https://orcid.org/0000-0001-6798-9683"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Danyang Xiao","raw_affiliation_strings":["Sun Yat-sen University, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sun Yat-sen University, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062442860","display_name":"Mengqiang Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mengqiang Chen","raw_affiliation_strings":["Sun Yat-sen University, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sun Yat-sen University, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100918579","display_name":"Jieying Zhou","orcid":null},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jieying Zhou","raw_affiliation_strings":["Sun Yat-sen University, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sun Yat-sen University, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5084826798","display_name":"Weigang Wu","orcid":"https://orcid.org/0000-0002-4714-7021"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weigang Wu","raw_affiliation_strings":["Sun Yat-sen University, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sun Yat-sen University, China","institution_ids":["https://openalex.org/I157773358"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.8125,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.79166938,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10500","display_name":"Sparse and Compressive Sensing Techniques","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/asynchronous-communication","display_name":"Asynchronous communication","score":0.8549578189849854},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8546167016029358},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.6907862424850464},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.6493692994117737},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.6259692311286926},{"id":"https://openalex.org/keywords/distributed-learning","display_name":"Distributed learning","score":0.6229518055915833},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5834810137748718},{"id":"https://openalex.org/keywords/dual","display_name":"Dual (grammatical number)","score":0.546394944190979},{"id":"https://openalex.org/keywords/distributed-algorithm","display_name":"Distributed algorithm","score":0.5269380807876587},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5086541771888733},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4839227795600891},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.4823457598686218},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4766698181629181},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.4428308606147766},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.40761592984199524},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.38544291257858276},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.25646138191223145}],"concepts":[{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.8549578189849854},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8546167016029358},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.6907862424850464},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.6493692994117737},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.6259692311286926},{"id":"https://openalex.org/C2779582901","wikidata":"https://www.wikidata.org/wiki/Q21013010","display_name":"Distributed learning","level":2,"score":0.6229518055915833},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5834810137748718},{"id":"https://openalex.org/C2780980858","wikidata":"https://www.wikidata.org/wiki/Q110022","display_name":"Dual (grammatical number)","level":2,"score":0.546394944190979},{"id":"https://openalex.org/C130120984","wikidata":"https://www.wikidata.org/wiki/Q2835898","display_name":"Distributed algorithm","level":2,"score":0.5269380807876587},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5086541771888733},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4839227795600891},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.4823457598686218},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4766698181629181},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.4428308606147766},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40761592984199524},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.38544291257858276},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.25646138191223145},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C124952713","wikidata":"https://www.wikidata.org/wiki/Q8242","display_name":"Literature","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3404397.3404401","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404397.3404401","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"49th International Conference on Parallel Processing - ICPP","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W1867117070","https://openalex.org/W2086161653","https://openalex.org/W2087402357","https://openalex.org/W2108598243","https://openalex.org/W2117539524","https://openalex.org/W2154834860","https://openalex.org/W2168231600","https://openalex.org/W2405578611","https://openalex.org/W2407022425","https://openalex.org/W2617242334","https://openalex.org/W2760303966","https://openalex.org/W2884711234","https://openalex.org/W2950819300","https://openalex.org/W2963540381","https://openalex.org/W2963792515","https://openalex.org/W2963964896","https://openalex.org/W2975523263","https://openalex.org/W4230874317"],"related_works":["https://openalex.org/W4287755480","https://openalex.org/W2785875001","https://openalex.org/W4297791730","https://openalex.org/W2904308246","https://openalex.org/W3194994764","https://openalex.org/W2799640661","https://openalex.org/W4210558419","https://openalex.org/W4363649046","https://openalex.org/W2151520847","https://openalex.org/W2953763514"],"abstract_inverted_index":{"Distributed":[0],"parallel":[1],"training":[2,21,34],"using":[3],"computing":[4],"clusters":[5],"is":[6,22],"desirable":[7],"for":[8,26],"large":[9],"scale":[10],"deep":[11],"neural":[12],"networks.":[13],"One":[14],"of":[15,47],"the":[16,23,45,53],"key":[17],"challenges":[18],"in":[19,70],"distributed":[20,72],"communication":[24],"cost":[25],"exchanging":[27],"information,":[28],"such":[29],"as":[30],"stochastic":[31],"gradients,":[32],"among":[33],"nodes.":[35],"Recently,":[36],"gradient":[37,59],"sparsification":[38,60],"techniques":[39],"have":[40],"been":[41],"proposed":[42],"to":[43],"reduce":[44],"amount":[46],"data":[48],"exchanged":[49],"and":[50,66],"thus":[51],"alleviate":[52],"network":[54],"overhead.":[55],"However,":[56],"most":[57],"existing":[58],"approaches":[61],"consider":[62],"only":[63],"synchronous":[64],"parallelism":[65],"cannot":[67],"be":[68],"applied":[69],"asynchronous":[71],"training.":[73]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}