{"id":"https://openalex.org/W4213132668","doi":"https://doi.org/10.1109/tpds.2022.3151739","title":"Predicting Throughput of Distributed Stochastic Gradient Descent","display_name":"Predicting Throughput of Distributed Stochastic Gradient Descent","publication_year":2022,"publication_date":"2022-02-16","ids":{"openalex":"https://openalex.org/W4213132668","doi":"https://doi.org/10.1109/tpds.2022.3151739"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2022.3151739","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2022.3151739","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013473165","display_name":"Zhuojin Li","orcid":"https://orcid.org/0000-0002-8308-0231"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhuojin Li","raw_affiliation_strings":["Department of Computer Science, University of Southern California, Los Angeles, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-8308-0231","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036380703","display_name":"Marco Paolieri","orcid":"https://orcid.org/0000-0001-5110-203X"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Marco Paolieri","raw_affiliation_strings":["Department of Computer Science, University of Southern California, Los Angeles, CA, USA"],"raw_orcid":"https://orcid.org/0000-0001-5110-203X","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081582862","display_name":"Leana Golubchik","orcid":"https://orcid.org/0000-0001-8353-5040"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Leana Golubchik","raw_affiliation_strings":["Department of Computer Science, University of Southern California, Los Angeles, CA, USA"],"raw_orcid":"https://orcid.org/0000-0001-8353-5040","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045463992","display_name":"Sung-Han Lin","orcid":"https://orcid.org/0000-0001-5666-9393"},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sung-Han Lin","raw_affiliation_strings":["Meta, Menlo Park, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210099336"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5003283573","display_name":"Wumo Yan","orcid":"https://orcid.org/0000-0002-4506-795X"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wumo Yan","raw_affiliation_strings":["Department of Computer Science, University of Southern California, Los Angeles, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-4506-795X","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.4867,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.57727572,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":97},"biblio":{"volume":"33","issue":"11","first_page":"2900","last_page":"2912"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8525265455245972},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.7468855381011963},{"id":"https://openalex.org/keywords/asynchronous-communication","display_name":"Asynchronous communication","score":0.631989598274231},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.4993467330932617},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.47384363412857056},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.44557759165763855},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.4421067237854004},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.4210575222969055},{"id":"https://openalex.org/keywords/distributed-algorithm","display_name":"Distributed algorithm","score":0.4103657901287079},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.3515549898147583},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.3356659412384033},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.31701913475990295},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.18211281299591064},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.17736127972602844},{"id":"https://openalex.org/keywords/wireless","display_name":"Wireless","score":0.16316255927085876}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8525265455245972},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.7468855381011963},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.631989598274231},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.4993467330932617},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.47384363412857056},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.44557759165763855},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.4421067237854004},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4210575222969055},{"id":"https://openalex.org/C130120984","wikidata":"https://www.wikidata.org/wiki/Q2835898","display_name":"Distributed algorithm","level":2,"score":0.4103657901287079},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3515549898147583},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3356659412384033},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31701913475990295},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.18211281299591064},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.17736127972602844},{"id":"https://openalex.org/C555944384","wikidata":"https://www.wikidata.org/wiki/Q249","display_name":"Wireless","level":2,"score":0.16316255927085876},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2022.3151739","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2022.3151739","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure","score":0.46000000834465027}],"awards":[{"id":"https://openalex.org/G529689006","display_name":null,"funder_award_id":"CNS-1816887","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7728769296","display_name":null,"funder_award_id":"CCF-1763747","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":49,"referenced_works":["https://openalex.org/W1978006658","https://openalex.org/W1982063824","https://openalex.org/W2057332538","https://openalex.org/W2097117768","https://openalex.org/W2112684994","https://openalex.org/W2132737349","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2309679942","https://openalex.org/W2336650964","https://openalex.org/W2436522418","https://openalex.org/W2606722458","https://openalex.org/W2622263826","https://openalex.org/W2752512710","https://openalex.org/W2796649226","https://openalex.org/W2798515322","https://openalex.org/W2884711234","https://openalex.org/W2899713699","https://openalex.org/W2919115771","https://openalex.org/W2920798074","https://openalex.org/W2946408574","https://openalex.org/W2953384591","https://openalex.org/W2953498285","https://openalex.org/W2955425717","https://openalex.org/W2962863496","https://openalex.org/W2963149401","https://openalex.org/W2963446712","https://openalex.org/W2963865397","https://openalex.org/W2964324519","https://openalex.org/W2972087877","https://openalex.org/W2987203808","https://openalex.org/W3086105743","https://openalex.org/W3105739685","https://openalex.org/W4295312788","https://openalex.org/W6679393576","https://openalex.org/W6697698479","https://openalex.org/W6703420464","https://openalex.org/W6713134421","https://openalex.org/W6718307486","https://openalex.org/W6739622702","https://openalex.org/W6744307745","https://openalex.org/W6745458143","https://openalex.org/W6750448596","https://openalex.org/W6753209298","https://openalex.org/W6760272376","https://openalex.org/W6762718338","https://openalex.org/W6764791836","https://openalex.org/W6766978945","https://openalex.org/W6780043775"],"related_works":["https://openalex.org/W2280422768","https://openalex.org/W3143197806","https://openalex.org/W4252555497","https://openalex.org/W2116677773","https://openalex.org/W3121175838","https://openalex.org/W2109294620","https://openalex.org/W4283028824","https://openalex.org/W2766289720","https://openalex.org/W1528719522","https://openalex.org/W3194994764"],"abstract_inverted_index":{"Training":[0],"jobs":[1],"of":[2,13,30,77,83,108,147],"deep":[3],"neural":[4],"networks":[5],"(DNNs)":[6],"can":[7,60,140,152,166],"be":[8,141,167],"accelerated":[9],"through":[10],"distributed":[11,84,148],"variants":[12,146],"stochastic":[14],"gradient":[15],"descent":[16],"(SGD),":[17],"where":[18,97],"multiple":[19],"nodes":[20,32,67],"process":[21],"training":[22],"examples":[23],"and":[24,46,58,92,115,124,139,163],"exchange":[25],"updates.":[26],"The":[27],"total":[28],"throughput":[29,82],"the":[31],"depends":[33],"not":[34],"only":[35],"on":[36,42],"their":[37,43,93,164],"computing":[38],"power,":[39],"but":[40],"also":[41],"networking":[44],"speeds":[45],"coordination":[47],"mechanism":[48],"(synchronous":[49],"or":[50,53],"asynchronous,":[51],"centralized":[52],"decentralized),":[54],"since":[55],"communication":[56],"bottlenecks":[57],"stragglers":[59],"result":[61],"in":[62],"sublinear":[63],"scaling":[64],"when":[65,157],"additional":[66],"are":[68,103,136],"provisioned.":[69],"In":[70],"this":[71],"paper,":[72],"we":[73],"propose":[74],"two":[75],"classes":[76],"performance":[78],"models":[79,123,135,151],"to":[80,144,175],"predict":[81],"SGD:fine-grained":[85],"models,":[86,96],"representing":[87],"many":[88],"elementary":[89],"computation/communication":[90],"operations":[91],"dependencies;":[94],"andcoarse-grained":[95],"SGD":[98],"steps":[99],"at":[100],"each":[101],"node":[102],"represented":[104],"as":[105],"a":[106,118],"sequence":[107],"high-level":[109],"phases":[110],"without":[111],"parallelism":[112],"between":[113],"computation":[114],"communication.":[116],"Using":[117],"PyTorch":[119],"implementation,":[120],"real-world":[121],"DNN":[122],"different":[125],"cloud":[126],"environments,":[127],"our":[128],"experimental":[129],"evaluation":[130],"illustrates":[131],"that,":[132],"while":[133],"fine-grained":[134],"more":[137],"accurate":[138,155],"easily":[142],"adapted":[143],"new":[145],"SGD,":[149],"coarse-grained":[150],"provide":[153],"similarly":[154],"predictions":[156],"augmented":[158],"with":[159,169],"ad":[160],"hoc":[161],"heuristics,":[162],"parameters":[165],"estimated":[168],"profiling":[170],"information":[171],"that":[172],"is":[173],"easier":[174],"collect.":[176]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":2}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-10-10T00:00:00"}