{"id":"https://openalex.org/W3021675794","doi":"https://doi.org/10.1109/tpds.2020.3040606","title":"Breaking (Global) Barriers in Parallel Stochastic Optimization with Wait-Avoiding Group Averaging","display_name":"Breaking (Global) Barriers in Parallel Stochastic Optimization with Wait-Avoiding Group Averaging","publication_year":2020,"publication_date":"2020-01-01","ids":{"openalex":"https://openalex.org/W3021675794","doi":"https://doi.org/10.1109/tpds.2020.3040606","mag":"3021675794"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2020.3040606","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2020.3040606","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2005.00124","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Shigang Li","orcid":null},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Shigang Li","raw_affiliation_strings":["Department of Computer Science, ETH Zurich, Z\u00fcrich, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, ETH Zurich, Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Tal Ben-Nun","orcid":null},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Tal Ben-Nun","raw_affiliation_strings":["Department of Computer Science, ETH Zurich, Z\u00fcrich, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, ETH Zurich, Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Giorgi Nadiradze","orcid":null},"institutions":[{"id":"https://openalex.org/I157556583","display_name":"Institute of Science and Technology Austria","ror":"https://ror.org/03gnh5541","country_code":"AT","type":"education","lineage":["https://openalex.org/I157556583"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Giorgi Nadiradze","raw_affiliation_strings":["IST Austria, Klosterneuburg, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IST Austria, Klosterneuburg, Austria","institution_ids":["https://openalex.org/I157556583"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Salvatore Digirolamo","orcid":null},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Salvatore Digirolamo","raw_affiliation_strings":["Department of Computer Science, ETH Zurich, Z\u00fcrich, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, ETH Zurich, Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Nikoli Dryden","orcid":null},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Nikoli Dryden","raw_affiliation_strings":["Department of Computer Science, ETH Zurich, Z\u00fcrich, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, ETH Zurich, Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Dan Alistarh","orcid":null},"institutions":[{"id":"https://openalex.org/I157556583","display_name":"Institute of Science and Technology Austria","ror":"https://ror.org/03gnh5541","country_code":"AT","type":"education","lineage":["https://openalex.org/I157556583"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Dan Alistarh","raw_affiliation_strings":["IST Austria, Klosterneuburg, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IST Austria, Klosterneuburg, Austria","institution_ids":["https://openalex.org/I157556583"]}]},{"author_position":"last","author":{"id":null,"display_name":"Torsten Hoefler","orcid":null},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Torsten Hoefler","raw_affiliation_strings":["Department of Computer Science, ETH Zurich, Z\u00fcrich, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, ETH Zurich, Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2608,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.61946599,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"1"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.5823000073432922,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.5823000073432922,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.11309999972581863,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.056299999356269836,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.70660001039505},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.6320000290870667},{"id":"https://openalex.org/keywords/stochastic-approximation","display_name":"Stochastic approximation","score":0.5024999976158142},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.4212000072002411},{"id":"https://openalex.org/keywords/rate-of-convergence","display_name":"Rate of convergence","score":0.37880000472068787},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.3772999942302704},{"id":"https://openalex.org/keywords/stochastic-optimization","display_name":"Stochastic optimization","score":0.36550000309944153},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.34850001335144043}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7809000015258789},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.70660001039505},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.6320000290870667},{"id":"https://openalex.org/C55479107","wikidata":"https://www.wikidata.org/wiki/Q97663916","display_name":"Stochastic approximation","level":3,"score":0.5024999976158142},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.4212000072002411},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.3977000117301941},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39500001072883606},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.37880000472068787},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.3772999942302704},{"id":"https://openalex.org/C194387892","wikidata":"https://www.wikidata.org/wiki/Q1747770","display_name":"Stochastic optimization","level":2,"score":0.36550000309944153},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.34850001335144043},{"id":"https://openalex.org/C8272713","wikidata":"https://www.wikidata.org/wiki/Q176737","display_name":"Stochastic process","level":2,"score":0.3314000070095062},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.33009999990463257},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3294999897480011},{"id":"https://openalex.org/C148764684","wikidata":"https://www.wikidata.org/wiki/Q621751","display_name":"Approximation algorithm","level":2,"score":0.3228999972343445},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32269999384880066},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.2957000136375427},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.29350000619888306},{"id":"https://openalex.org/C127491075","wikidata":"https://www.wikidata.org/wiki/Q7617825","display_name":"Stochastic modelling","level":2,"score":0.2736000120639801},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.27079999446868896},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2685999870300293},{"id":"https://openalex.org/C2779582901","wikidata":"https://www.wikidata.org/wiki/Q21013010","display_name":"Distributed learning","level":2,"score":0.26260000467300415},{"id":"https://openalex.org/C555944384","wikidata":"https://www.wikidata.org/wiki/Q249","display_name":"Wireless","level":2,"score":0.25130000710487366}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpds.2020.3040606","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2020.3040606","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2005.00124","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2005.00124","pdf_url":"https://arxiv.org/pdf/2005.00124","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2005.00124","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2005.00124","pdf_url":"https://arxiv.org/pdf/2005.00124","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4387927307","display_name":null,"funder_award_id":"Ambizione Project No. 185778","funder_id":"https://openalex.org/F4320320924","funder_display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung"}],"funders":[{"id":"https://openalex.org/F4320320924","display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung","ror":"https://ror.org/00yjd3n13"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":68,"referenced_works":["https://openalex.org/W1572016165","https://openalex.org/W1938987580","https://openalex.org/W2043143850","https://openalex.org/W2056476515","https://openalex.org/W2057332538","https://openalex.org/W2060393849","https://openalex.org/W2064675550","https://openalex.org/W2091765165","https://openalex.org/W2104171511","https://openalex.org/W2108598243","https://openalex.org/W2165808215","https://openalex.org/W2194775991","https://openalex.org/W2405883473","https://openalex.org/W2535838896","https://openalex.org/W2755567635","https://openalex.org/W2766672944","https://openalex.org/W2952046647","https://openalex.org/W2953127211","https://openalex.org/W2962780982","https://openalex.org/W2963433607","https://openalex.org/W2963470657","https://openalex.org/W2963964896","https://openalex.org/W2964339842","https://openalex.org/W2965658867","https://openalex.org/W2967558351","https://openalex.org/W2971624117","https://openalex.org/W2972087877","https://openalex.org/W2973793552","https://openalex.org/W2982316857","https://openalex.org/W2985108934","https://openalex.org/W2999044305","https://openalex.org/W3008591352","https://openalex.org/W3009928773","https://openalex.org/W3015640161","https://openalex.org/W3040257343","https://openalex.org/W3130294722","https://openalex.org/W6638803421","https://openalex.org/W6679393576","https://openalex.org/W6680402377","https://openalex.org/W6681545434","https://openalex.org/W6684249991","https://openalex.org/W6684859321","https://openalex.org/W6692488971","https://openalex.org/W6692846177","https://openalex.org/W6704559304","https://openalex.org/W6728757088","https://openalex.org/W6730009442","https://openalex.org/W6736413256","https://openalex.org/W6738250615","https://openalex.org/W6739901393","https://openalex.org/W6740154879","https://openalex.org/W6745458143","https://openalex.org/W6748645090","https://openalex.org/W6748948743","https://openalex.org/W6749257597","https://openalex.org/W6752191696","https://openalex.org/W6752515464","https://openalex.org/W6755251367","https://openalex.org/W6756592394","https://openalex.org/W6756976665","https://openalex.org/W6757043007","https://openalex.org/W6757053730","https://openalex.org/W6758557334","https://openalex.org/W6762211661","https://openalex.org/W6766978945","https://openalex.org/W6772383348","https://openalex.org/W6773029903","https://openalex.org/W6775649616"],"related_works":[],"abstract_inverted_index":{"Deep":[0],"learning":[1,125],"at":[2,128],"scale":[3],"is":[4,76],"dominated":[5],"by":[6],"communication":[7,68],"time.":[8],"Distributing":[9],"samples":[10],"across":[11,30],"nodes":[12],"usually":[13],"yields":[14],"the":[15,38,46,83,87,96,151,155,159],"best":[16],"performance,":[17],"but":[18,40],"poses":[19],"scaling":[20],"challenges":[21],"due":[22],"to":[23,44,82,109],"global":[24,67],"information":[25],"dissemination":[26],"and":[27,86,100,122,149],"load":[28],"imbalance":[29],"uneven":[31],"sample":[32],"lengths.":[33],"State-of-the-art":[34],"decentralized":[35,133],"optimizers":[36],"mitigate":[37],"problem,":[39],"require":[41],"more":[42],"iterations":[43],"achieve":[45],"same":[47],"accuracy":[48],"as":[49],"their":[50],"globally-communicating":[51],"counterparts.":[52],"We":[53,94],"present":[54],"Wait-Avoiding":[55],"Group":[56],"Model":[57],"Averaging":[58],"(WAGMA)":[59],"SGD,":[60],"a":[61,77,90],"wait-avoiding":[62],"stochastic":[63],"optimizer":[64],"that":[65,103],"reduces":[66],"via":[69],"subgroup":[70],"weight":[71],"exchange.":[72],"The":[73],"key":[74],"insight":[75],"combination":[78],"of":[79,89,98],"algorithmic":[80],"changes":[81],"averaging":[84],"scheme":[85],"use":[88],"group":[91],"allreduce":[92],"operation.":[93],"prove":[95],"convergence":[97,106],"WAGMA-SGD,":[99],"empirically":[101],"show":[102],"it":[104],"retains":[105],"rates":[107],"similar":[108],"Allreduce-SGD.":[110],"For":[111],"evaluation,":[112],"we":[113],"train":[114],"ResNet-50":[115],"on":[116,143],"ImageNet;":[117],"Transformer":[118],"for":[119,126,146,163],"machine":[120],"translation;":[121],"deep":[123],"reinforcement":[124,147],"navigation":[127],"scale.":[129],"Compared":[130],"with":[131],"state-of-the-art":[132],"SGD":[134],"variants,":[135],"WAGMA-SGD":[136],"significantly":[137],"improves":[138],"training":[139,161],"throughput":[140],"(e.g.,":[141,154],"2.1\u00d7":[142],"1,024":[144],"GPUs":[145],"learning),":[148],"achieves":[150],"fastest":[152],"time-to-solution":[153],"highest":[156],"score":[157],"using":[158],"shortest":[160],"time":[162],"Transformer).":[164]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2021,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2020-05-13T00:00:00"}