{"id":"https://openalex.org/W3204434815","doi":"https://doi.org/10.1109/icassp43922.2022.9747408","title":"S2 Reducer: High-Performance Sparse Communication to Accelerate Distributed Deep Learning","display_name":"S2 Reducer: High-Performance Sparse Communication to Accelerate Distributed Deep Learning","publication_year":2022,"publication_date":"2022-04-27","ids":{"openalex":"https://openalex.org/W3204434815","doi":"https://doi.org/10.1109/icassp43922.2022.9747408","mag":"3204434815"},"language":"en","primary_location":{"id":"doi:10.1109/icassp43922.2022.9747408","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp43922.2022.9747408","pdf_url":null,"source":{"id":"https://openalex.org/S4363607702","display_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074708780","display_name":"Keshi Ge","orcid":"https://orcid.org/0000-0002-0669-6892"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Keshi Ge","raw_affiliation_strings":["National University of Defense Technology,College of Computer","College of Computer, National University of Defense Technology"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology,College of Computer","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"College of Computer, National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003730869","display_name":"Yongquan Fu","orcid":"https://orcid.org/0000-0002-7564-5239"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yongquan Fu","raw_affiliation_strings":["National University of Defense Technology,College of Computer","College of Computer, National University of Defense Technology"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology,College of Computer","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"College of Computer, National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100395351","display_name":"Yiming Zhang","orcid":"https://orcid.org/0000-0001-6450-8485"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yiming Zhang","raw_affiliation_strings":["National University of Defense Technology,College of Computer","College of Computer, National University of Defense Technology"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology,College of Computer","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"College of Computer, National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055426388","display_name":"Zhiquan Lai","orcid":"https://orcid.org/0000-0002-3458-4732"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiquan Lai","raw_affiliation_strings":["National University of Defense Technology,College of Computer","College of Computer, National University of Defense Technology"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology,College of Computer","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"College of Computer, National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001375179","display_name":"Xiaoge Deng","orcid":null},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoge Deng","raw_affiliation_strings":["National University of Defense Technology,College of Computer","College of Computer, National University of Defense Technology"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology,College of Computer","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"College of Computer, National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100440919","display_name":"Dong\u2010Sheng Li","orcid":"https://orcid.org/0000-0003-1283-6334"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongsheng Li","raw_affiliation_strings":["National University of Defense Technology,College of Computer","College of Computer, National University of Defense Technology"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology,College of Computer","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"College of Computer, National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5074708780"],"corresponding_institution_ids":["https://openalex.org/I170215575"],"apc_list":null,"apc_paid":null,"fwci":0.2998,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.61321831,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"5233","last_page":"5237"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reducer","display_name":"Reducer","score":0.8348658680915833},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8144749402999878},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.792147696018219},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7748643159866333},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.697510302066803},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.5606997013092041},{"id":"https://openalex.org/keywords/bandwidth","display_name":"Bandwidth (computing)","score":0.5396353006362915},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.5244746208190918},{"id":"https://openalex.org/keywords/sketch","display_name":"Sketch","score":0.4976384937763214},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.48086434602737427},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.47791701555252075},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.4284557104110718},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4024762809276581},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.350340873003006},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.3428914546966553},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.24093693494796753},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.1566809117794037},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.11959406733512878},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.07121589779853821}],"concepts":[{"id":"https://openalex.org/C2776985865","wikidata":"https://www.wikidata.org/wiki/Q26820931","display_name":"Reducer","level":2,"score":0.8348658680915833},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8144749402999878},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.792147696018219},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7748643159866333},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.697510302066803},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5606997013092041},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.5396353006362915},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.5244746208190918},{"id":"https://openalex.org/C2779231336","wikidata":"https://www.wikidata.org/wiki/Q7534724","display_name":"Sketch","level":2,"score":0.4976384937763214},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.48086434602737427},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.47791701555252075},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.4284557104110718},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4024762809276581},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.350340873003006},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.3428914546966553},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.24093693494796753},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.1566809117794037},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.11959406733512878},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.07121589779853821},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp43922.2022.9747408","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp43922.2022.9747408","pdf_url":null,"source":{"id":"https://openalex.org/S4363607702","display_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":46,"referenced_works":["https://openalex.org/W2060393849","https://openalex.org/W2083842231","https://openalex.org/W2194775991","https://openalex.org/W2294895103","https://openalex.org/W2402144811","https://openalex.org/W2525778437","https://openalex.org/W2769644379","https://openalex.org/W2774000609","https://openalex.org/W2788193959","https://openalex.org/W2906007643","https://openalex.org/W2911863041","https://openalex.org/W2946900926","https://openalex.org/W2950826569","https://openalex.org/W2953384591","https://openalex.org/W2963446712","https://openalex.org/W2963803379","https://openalex.org/W2964110616","https://openalex.org/W2970289928","https://openalex.org/W2970421227","https://openalex.org/W2970971581","https://openalex.org/W2972087877","https://openalex.org/W2975712713","https://openalex.org/W2985108934","https://openalex.org/W3025165719","https://openalex.org/W3081168214","https://openalex.org/W3091097978","https://openalex.org/W3094502228","https://openalex.org/W3097777922","https://openalex.org/W3119786062","https://openalex.org/W3122154272","https://openalex.org/W3132107458","https://openalex.org/W3160525311","https://openalex.org/W3204434815","https://openalex.org/W4288417148","https://openalex.org/W4295312788","https://openalex.org/W6713134421","https://openalex.org/W6727690538","https://openalex.org/W6746200960","https://openalex.org/W6746839373","https://openalex.org/W6758358981","https://openalex.org/W6761040187","https://openalex.org/W6763138067","https://openalex.org/W6766978945","https://openalex.org/W6784333009","https://openalex.org/W6788684253","https://openalex.org/W6802173074"],"related_works":["https://openalex.org/W1800827217","https://openalex.org/W2499279132","https://openalex.org/W34241620","https://openalex.org/W4292761019","https://openalex.org/W2364921833","https://openalex.org/W2950720137","https://openalex.org/W2022425454","https://openalex.org/W1509211761","https://openalex.org/W4307961379","https://openalex.org/W3087165870"],"abstract_inverted_index":{"Distributed":[0],"stochastic":[1],"gradient":[2,16,67,83,96],"descent":[3],"(SGD)":[4],"approach":[5],"has":[6,37],"been":[7,38],"widely":[8,39],"used":[9],"in":[10,62],"large-scale":[11],"deep":[12,29],"learning,":[13],"and":[14,71,116,118,156],"the":[15,23,27,42,48,81,105,111,120,148],"collective":[17],"method":[18,98],"is":[19],"vital":[20],"to":[21,46,147,163],"ensure":[22],"training":[24,138,160],"scalability":[25],"of":[26],"distributed":[28,43,159],"learning":[30],"system.":[31],"Collective":[32],"communication":[33,49,84,106,154],"such":[34],"as":[35],"AllReduce":[36,52,122],"adopted":[40],"for":[41,76,124],"SGD":[44,126],"process":[45],"reduce":[47,80],"time.":[50],"However,":[51],"incurs":[53],"large":[54],"bandwidth":[55,77],"resources":[56],"while":[57],"most":[58],"gradients":[59,113],"are":[60,69],"sparse":[61,82,95,153],"many":[63,66],"cases":[64],"since":[65],"values":[68],"zeros":[70],"should":[72],"be":[73],"efficiently":[74],"compressed":[75],"saving.":[78],"To":[79],"overhead,":[85,155],"we":[86],"propose":[87],"Sparse-Sketch":[88],"Reducer":[89,103,145],"(S2":[90],"Reducer),":[91],"a":[92],"novel":[93],"sketch-based":[94],"aggregation":[97],"with":[99,114],"convergence":[100],"guarantees.":[101],"S2":[102,144],"reduces":[104,151],"cost":[107],"by":[108],"only":[109],"compressing":[110],"non-zero":[112],"count-sketch":[115],"bitmap,":[117],"enables":[119],"efficient":[121],"operators":[123],"parallel":[125],"training.":[127],"We":[128],"perform":[129],"extensive":[130],"evaluation":[131],"against":[132],"four":[133],"state-of-the-art":[134,164],"methods":[135],"over":[136],"five":[137],"models.":[139],"Our":[140],"results":[141],"show":[142],"that":[143],"converges":[146],"same":[149],"accuracy,":[150],"81%":[152],"achieves":[157],"1.8\u00d7":[158],"speedup":[161],"compared":[162],"approaches.":[165]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}