{"id":"https://openalex.org/W2967558351","doi":"https://doi.org/10.1145/3332466.3374528","title":"Taming unbalanced training workloads in deep learning with partial collective operations","display_name":"Taming unbalanced training workloads in deep learning with partial collective operations","publication_year":2020,"publication_date":"2020-02-19","ids":{"openalex":"https://openalex.org/W2967558351","doi":"https://doi.org/10.1145/3332466.3374528","mag":"2967558351"},"language":"en","primary_location":{"id":"doi:10.1145/3332466.3374528","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3332466.3374528","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 25th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1908.04207","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088172355","display_name":"Shigang Li","orcid":"https://orcid.org/0000-0003-0022-7865"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Shigang Li","raw_affiliation_strings":["ETH Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002615744","display_name":"Tal Ben\u2010Nun","orcid":"https://orcid.org/0000-0002-3657-6568"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Tal Ben-Nun","raw_affiliation_strings":["ETH Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019905167","display_name":"Salvatore Di Girolamo","orcid":"https://orcid.org/0000-0003-2197-8860"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Salvatore Di Girolamo","raw_affiliation_strings":["ETH Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083822059","display_name":"Dan Alistarh","orcid":"https://orcid.org/0000-0003-3650-940X"},"institutions":[{"id":"https://openalex.org/I157556583","display_name":"Institute of Science and Technology Austria","ror":"https://ror.org/03gnh5541","country_code":"AT","type":"education","lineage":["https://openalex.org/I157556583"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Dan Alistarh","raw_affiliation_strings":["IST Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IST Austria","institution_ids":["https://openalex.org/I157556583"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5026990786","display_name":"Torsten Hoefler","orcid":"https://orcid.org/0000-0002-1333-9797"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Torsten Hoefler","raw_affiliation_strings":["ETH Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich","institution_ids":["https://openalex.org/I35440088"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":55,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"45","last_page":"61"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12702","display_name":"Brain Tumor Detection and Classification","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/2808","display_name":"Neurology"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.9310941696166992},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.8280277252197266},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8109321594238281},{"id":"https://openalex.org/keywords/asynchronous-communication","display_name":"Asynchronous communication","score":0.7856974601745605},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.6504030823707581},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.6487728357315063},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5616886615753174},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.5566467046737671},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.4904632270336151},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4511137306690216},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.44060957431793213},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.435829758644104},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.3443041443824768},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.3090495765209198},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.12368839979171753}],"concepts":[{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.9310941696166992},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.8280277252197266},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8109321594238281},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.7856974601745605},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.6504030823707581},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.6487728357315063},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5616886615753174},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.5566467046737671},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.4904632270336151},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4511137306690216},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.44060957431793213},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.435829758644104},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3443041443824768},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3090495765209198},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.12368839979171753},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3332466.3374528","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3332466.3374528","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 25th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1908.04207","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1908.04207","pdf_url":"https://arxiv.org/pdf/1908.04207","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1908.04207","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1908.04207","pdf_url":"https://arxiv.org/pdf/1908.04207","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":73,"referenced_works":["https://openalex.org/W24089286","https://openalex.org/W1442374986","https://openalex.org/W1570056235","https://openalex.org/W1572016165","https://openalex.org/W1686810756","https://openalex.org/W1923404803","https://openalex.org/W1938987580","https://openalex.org/W1994616650","https://openalex.org/W2057332538","https://openalex.org/W2060393849","https://openalex.org/W2062011830","https://openalex.org/W2064675550","https://openalex.org/W2083842231","https://openalex.org/W2091765165","https://openalex.org/W2104171511","https://openalex.org/W2108598243","https://openalex.org/W2112796928","https://openalex.org/W2131613942","https://openalex.org/W2132737349","https://openalex.org/W2136836265","https://openalex.org/W2138243089","https://openalex.org/W2163349370","https://openalex.org/W2165808215","https://openalex.org/W2168231600","https://openalex.org/W2168695393","https://openalex.org/W2180092181","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2221532093","https://openalex.org/W2259324379","https://openalex.org/W2405578611","https://openalex.org/W2407022425","https://openalex.org/W2508429489","https://openalex.org/W2553581924","https://openalex.org/W2568564101","https://openalex.org/W2573285109","https://openalex.org/W2580688187","https://openalex.org/W2604783387","https://openalex.org/W2617960902","https://openalex.org/W2746314669","https://openalex.org/W2755567635","https://openalex.org/W2787998955","https://openalex.org/W2788193959","https://openalex.org/W2789218400","https://openalex.org/W2790598069","https://openalex.org/W2886189612","https://openalex.org/W2896457183","https://openalex.org/W2914304175","https://openalex.org/W2919115771","https://openalex.org/W2951183276","https://openalex.org/W2952046647","https://openalex.org/W2953000681","https://openalex.org/W2962747323","https://openalex.org/W2962863496","https://openalex.org/W2962950660","https://openalex.org/W2963149042","https://openalex.org/W2963228337","https://openalex.org/W2963403868","https://openalex.org/W2963446712","https://openalex.org/W2963766684","https://openalex.org/W2963964896","https://openalex.org/W2971624117","https://openalex.org/W3037338728","https://openalex.org/W4206638755","https://openalex.org/W4237284205","https://openalex.org/W4239072543","https://openalex.org/W4288357791","https://openalex.org/W4294527468","https://openalex.org/W4297688475","https://openalex.org/W4298840580","https://openalex.org/W4299802343","https://openalex.org/W4301239768","https://openalex.org/W4385245566"],"related_works":["https://openalex.org/W4246549241","https://openalex.org/W2168758875","https://openalex.org/W2410733619","https://openalex.org/W2963483475","https://openalex.org/W2955229517","https://openalex.org/W2966297898","https://openalex.org/W2120877146","https://openalex.org/W4206903459","https://openalex.org/W4283028824","https://openalex.org/W2754816816"],"abstract_inverted_index":{"Load":[0],"imbalance":[1,15],"pervasively":[2],"exists":[3],"in":[4,16,130],"distributed":[5],"deep":[6],"learning":[7],"training":[8,50],"systems,":[9],"either":[10],"caused":[11],"by":[12,20],"the":[13,21,46,60,82,92,103,120,123,127,159,178],"inherent":[14],"learned":[17],"tasks":[18],"or":[19],"system":[22],"itself.":[23],"Traditional":[24],"synchronous":[25,180],"Stochastic":[26],"Gradient":[27],"Descent":[28],"(SGD)":[29],"achieves":[30,151,164],"good":[31],"accuracy":[32],"for":[33,63,91],"a":[34,113,136],"wide":[35],"variety":[36,137],"of":[37,102,122,138],"tasks,":[38],"but":[39],"relies":[40],"on":[41,135,145,156,169,175],"global":[42,61],"synchronization":[43,62],"to":[44,71],"accumulate":[45],"gradients":[47,87,107],"at":[48,99],"every":[49],"step.":[51],"In":[52],"this":[53],"paper,":[54],"we":[55,69],"propose":[56,70],"eager-SGD,":[57,68],"which":[58],"relaxes":[59],"decentralized":[64,181],"accumulation.":[65],"To":[66],"implement":[67],"use":[72],"two":[73],"partial":[74,128],"collectives:":[75],"solo":[76,80],"and":[77,125,141,163,171],"majority.":[78],"With":[79],"allreduce,":[81,98],"faster":[83],"processes":[84],"contribute":[85,106],"their":[86],"eagerly":[88],"without":[89,111,183],"waiting":[90],"slower":[93],"processes,":[94],"whereas":[95],"with":[96],"majority":[97],"least":[100],"half":[101],"participants":[104],"must":[105],"before":[108],"continuing,":[109],"all":[110],"using":[112],"central":[114],"parameter":[115],"server.":[116],"We":[117],"theoretically":[118],"prove":[119],"convergence":[121],"algorithms":[124],"describe":[126],"collectives":[129],"detail.":[131],"Experiments":[132],"are":[133],"conducted":[134],"neural":[139],"networks":[140],"datasets.":[142],"The":[143],"results":[144],"load-imbalanced":[146],"environments":[147],"show":[148],"that":[149],"eager-SGD":[150],"2.64":[152],"X":[153,166],"speedup":[154,167,173],"(ResNet-50":[155,168],"ImageNet)":[157,170],"over":[158,177],"asynchronous":[160],"centralized":[161],"SGD,":[162],"1.29":[165],"1.27X":[172],"(LSTM":[174],"UCF101)":[176],"state-of-the-art":[179],"SGDs,":[182],"losing":[184],"accuracy.":[185]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":10},{"year":2021,"cited_by_count":14},{"year":2020,"cited_by_count":9}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}