{"id":"https://openalex.org/W4321487996","doi":"https://doi.org/10.1109/jsac.2023.3242733","title":"Compressed Collective Sparse-Sketch for Distributed Data-Parallel Training of Deep Learning Models","display_name":"Compressed Collective Sparse-Sketch for Distributed Data-Parallel Training of Deep Learning Models","publication_year":2023,"publication_date":"2023-02-22","ids":{"openalex":"https://openalex.org/W4321487996","doi":"https://doi.org/10.1109/jsac.2023.3242733"},"language":"en","primary_location":{"id":"doi:10.1109/jsac.2023.3242733","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jsac.2023.3242733","pdf_url":null,"source":{"id":"https://openalex.org/S90422530","display_name":"IEEE Journal on Selected Areas in Communications","issn_l":"0733-8716","issn":["0733-8716","1558-0008"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal on Selected Areas in Communications","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074708780","display_name":"Keshi Ge","orcid":"https://orcid.org/0000-0002-0669-6892"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Keshi Ge","raw_affiliation_strings":["College of Computer, National University of Defense Technology, Changsha, China"],"affiliations":[{"raw_affiliation_string":"College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100625354","display_name":"Kai L\u00fc","orcid":"https://orcid.org/0000-0002-6378-7002"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kai Lu","raw_affiliation_strings":["College of Computer, National University of Defense Technology, Changsha, China"],"affiliations":[{"raw_affiliation_string":"College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003730869","display_name":"Yongquan Fu","orcid":"https://orcid.org/0000-0002-7564-5239"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yongquan Fu","raw_affiliation_strings":["College of Computer, National University of Defense Technology, Changsha, China"],"affiliations":[{"raw_affiliation_string":"College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019549276","display_name":"Xiaoge Deng","orcid":"https://orcid.org/0000-0003-0622-1202"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoge Deng","raw_affiliation_strings":["College of Computer, National University of Defense Technology, Changsha, China"],"affiliations":[{"raw_affiliation_string":"College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055426388","display_name":"Zhiquan Lai","orcid":"https://orcid.org/0000-0002-3458-4732"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiquan Lai","raw_affiliation_strings":["College of Computer, National University of Defense Technology, Changsha, China"],"affiliations":[{"raw_affiliation_string":"College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100440903","display_name":"Dongsheng Li","orcid":"https://orcid.org/0000-0001-9743-2034"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongsheng Li","raw_affiliation_strings":["College of Computer, National University of Defense Technology, Changsha, China"],"affiliations":[{"raw_affiliation_string":"College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5074708780"],"corresponding_institution_ids":["https://openalex.org/I170215575"],"apc_list":null,"apc_paid":null,"fwci":1.0072,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.77083362,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":"41","issue":"4","first_page":"941","last_page":"963"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8204238414764404},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7159574627876282},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.6328779458999634},{"id":"https://openalex.org/keywords/sketch","display_name":"Sketch","score":0.6203406453132629},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.5603741407394409},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4950215220451355},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.46153461933135986},{"id":"https://openalex.org/keywords/hash-function","display_name":"Hash function","score":0.44825172424316406},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.42326292395591736},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.4184207320213318},{"id":"https://openalex.org/keywords/rate-of-convergence","display_name":"Rate of convergence","score":0.41156119108200073},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.3847999572753906},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3840175271034241},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.2514688968658447},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.14792275428771973},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.12648749351501465}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8204238414764404},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7159574627876282},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.6328779458999634},{"id":"https://openalex.org/C2779231336","wikidata":"https://www.wikidata.org/wiki/Q7534724","display_name":"Sketch","level":2,"score":0.6203406453132629},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.5603741407394409},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4950215220451355},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.46153461933135986},{"id":"https://openalex.org/C99138194","wikidata":"https://www.wikidata.org/wiki/Q183427","display_name":"Hash function","level":2,"score":0.44825172424316406},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.42326292395591736},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.4184207320213318},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.41156119108200073},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3847999572753906},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3840175271034241},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2514688968658447},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.14792275428771973},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.12648749351501465},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C555944384","wikidata":"https://www.wikidata.org/wiki/Q249","display_name":"Wireless","level":2,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/jsac.2023.3242733","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jsac.2023.3242733","pdf_url":null,"source":{"id":"https://openalex.org/S90422530","display_name":"IEEE Journal on Selected Areas in Communications","issn_l":"0733-8716","issn":["0733-8716","1558-0008"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal on Selected Areas in Communications","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3606055544","display_name":null,"funder_award_id":"62025208","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6014385352","display_name":null,"funder_award_id":"2021YFB0301200","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G7291170091","display_name":null,"funder_award_id":"61972409","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":59,"referenced_works":["https://openalex.org/W1972525637","https://openalex.org/W2039548408","https://openalex.org/W2057332538","https://openalex.org/W2083842231","https://openalex.org/W2095705004","https://openalex.org/W2194775991","https://openalex.org/W2294895103","https://openalex.org/W2405578611","https://openalex.org/W2512924740","https://openalex.org/W2525778437","https://openalex.org/W2606722458","https://openalex.org/W2906007643","https://openalex.org/W2935041335","https://openalex.org/W2963446712","https://openalex.org/W2964110616","https://openalex.org/W2966527647","https://openalex.org/W2972087877","https://openalex.org/W2975712713","https://openalex.org/W2985108934","https://openalex.org/W2995022099","https://openalex.org/W3081168214","https://openalex.org/W3091097978","https://openalex.org/W3097777922","https://openalex.org/W3118608800","https://openalex.org/W3132107458","https://openalex.org/W3160525311","https://openalex.org/W3173523152","https://openalex.org/W3204434815","https://openalex.org/W3206636350","https://openalex.org/W6628377381","https://openalex.org/W6638214083","https://openalex.org/W6674330103","https://openalex.org/W6684859321","https://openalex.org/W6713134421","https://openalex.org/W6727099177","https://openalex.org/W6727690538","https://openalex.org/W6738460352","https://openalex.org/W6739693220","https://openalex.org/W6741986022","https://openalex.org/W6742690309","https://openalex.org/W6745723224","https://openalex.org/W6746062806","https://openalex.org/W6746200960","https://openalex.org/W6746839373","https://openalex.org/W6748019269","https://openalex.org/W6748373249","https://openalex.org/W6754416507","https://openalex.org/W6754776447","https://openalex.org/W6754930508","https://openalex.org/W6758358981","https://openalex.org/W6761040187","https://openalex.org/W6763138067","https://openalex.org/W6766978945","https://openalex.org/W6779739340","https://openalex.org/W6780897730","https://openalex.org/W6784072064","https://openalex.org/W6786422851","https://openalex.org/W6787972765","https://openalex.org/W6788684253"],"related_works":["https://openalex.org/W2058965144","https://openalex.org/W2164382479","https://openalex.org/W2378994405","https://openalex.org/W2385974820","https://openalex.org/W2146343568","https://openalex.org/W98480971","https://openalex.org/W2150291671","https://openalex.org/W2013643406","https://openalex.org/W2371326241","https://openalex.org/W1966837078"],"abstract_inverted_index":{"Distributed":[0],"data-parallel":[1,134],"training":[2,13,57,135],"(DDP)":[3],"is":[4],"prevalent":[5],"in":[6,118,210],"large-scale":[7],"deep":[8,168],"learning.":[9],"To":[10],"increase":[11],"the":[12,55,109,115,128,153,178,188],"throughput":[14],"and":[15,69,90,101,136,196],"scalability,":[16],"high-performance":[17],"collective":[18,83],"communication":[19,35,42,84,92,140,190],"methods":[20,165],"such":[21],"as":[22,132,181],"AllReduce":[23,68,105],"have":[24],"recently":[25],"proliferated":[26],"for":[27,67,152],"DDP":[28,156],"use.":[29],"However,":[30],"these":[31],"approaches":[32,61,185],"require":[33],"long":[34],"periods":[36],"with":[37],"increasing":[38],"model":[39],"sizes.":[40],"Collective":[41],"transmits":[43],"many":[44],"sparse":[45,76,189],"gradient":[46,88],"values":[47],"that":[48,124,174,182],"can":[49],"be":[50],"efficiently":[51],"compressed":[52],"to":[53,103,113,177,194,202],"reduce":[54],"required":[56],"time.":[58],"State-of-the-art":[59],"compression":[60,66],"do":[62],"not":[63],"provide":[64],"mergeable":[65],"lack":[70],"convergence":[71,130],"bounds.":[72],"We":[73,107,122,147,158],"present":[74],"a":[75,80,95,119,137,149,198],"sketch":[77,99,111],"reducer":[78],"(S2Reducer),":[79],"sparsity-preserving":[81],"sketch-based":[82],"method.":[85],"S2Reducer":[86,151,175],"preserves":[87],"sparsity":[89],"reduces":[91],"costs":[93],"via":[94],"bitmap":[96],"informed":[97],"count":[98,110],"structure":[100],"adapts":[102],"efficient":[104],"operators.":[106],"tune":[108],"organization":[112],"minimize":[114],"hash":[116],"conflicts":[117],"fixed-size":[120],"budget.":[121],"prove":[123],"our":[125],"method":[126],"has":[127],"same":[129,179],"rate":[131],"vanilla":[133],"much":[138],"smaller":[139],"overhead":[141,191],"than":[142],"those":[143],"of":[144,183,200],"state-of-the-art":[145,164,184],"methods.":[146],"implement":[148],"GPU-accelerated":[150],"Ring":[154],"AllReduce-based":[155],"system.":[157],"perform":[159],"extensive":[160],"evaluations":[161],"against":[162],"four":[163],"across":[166],"seven":[167],"learning":[169],"models.":[170],"Our":[171],"results":[172],"show":[173],"converges":[176],"accuracy":[180],"while":[186],"reducing":[187],"by":[192],"up":[193,201],"86%":[195],"achieving":[197],"speedup":[199],"<inline-formula":[203],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[204],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">":[205],"<tex-math":[206],"notation=\"LaTeX\">$3.5\\times":[207],"$":[208],"</tex-math></inline-formula>":[209],"distributed":[211],"training.":[212]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}