{"id":"https://openalex.org/W3188065709","doi":"https://doi.org/10.1109/isca52012.2021.00049","title":"Enabling Compute-Communication Overlap in Distributed Deep Learning Training Platforms","display_name":"Enabling Compute-Communication Overlap in Distributed Deep Learning Training Platforms","publication_year":2021,"publication_date":"2021-06-01","ids":{"openalex":"https://openalex.org/W3188065709","doi":"https://doi.org/10.1109/isca52012.2021.00049","mag":"3188065709"},"language":"en","primary_location":{"id":"doi:10.1109/isca52012.2021.00049","is_oa":false,"landing_page_url":"https://doi.org/10.1109/isca52012.2021.00049","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 ACM/IEEE 48th Annual International Symposium on Computer Architecture (ISCA)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2007.00156","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088899364","display_name":"Saeed Rashidi","orcid":"https://orcid.org/0000-0002-6472-9920"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saeed Rashidi","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085294909","display_name":"Matthew Denton","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew Denton","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102919742","display_name":"Srinivas Sridharan","orcid":"https://orcid.org/0009-0001-0651-370X"},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Srinivas Sridharan","raw_affiliation_strings":["Facebook, Menlo Park, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, USA","institution_ids":["https://openalex.org/I4210099336","https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065143008","display_name":"Sudarshan Srinivasan","orcid":"https://orcid.org/0009-0002-8662-5820"},"institutions":[{"id":"https://openalex.org/I4210146682","display_name":"Intel (India)","ror":"https://ror.org/04f2n1245","country_code":"IN","type":"company","lineage":["https://openalex.org/I1343180700","https://openalex.org/I4210146682"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Sudarshan Srinivasan","raw_affiliation_strings":["Intel, Bangalore, India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Intel, Bangalore, India","institution_ids":["https://openalex.org/I4210146682"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103870027","display_name":"Amoghavarsha Suresh","orcid":"https://orcid.org/0009-0005-0999-6638"},"institutions":[{"id":"https://openalex.org/I59553526","display_name":"Stony Brook University","ror":"https://ror.org/05qghxh33","country_code":"US","type":"education","lineage":["https://openalex.org/I59553526"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amoghavarsha Suresh","raw_affiliation_strings":["Stony Brook University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stony Brook University, USA","institution_ids":["https://openalex.org/I59553526"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004002767","display_name":"Jade Nie","orcid":"https://orcid.org/0009-0008-1664-2805"},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jade Nie","raw_affiliation_strings":["Facebook, Menlo Park, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook, Menlo Park, USA","institution_ids":["https://openalex.org/I4210099336","https://openalex.org/I4210114444"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034089074","display_name":"Tushar Krishna","orcid":"https://orcid.org/0000-0001-5738-6942"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tushar Krishna","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, USA","institution_ids":["https://openalex.org/I130701444"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":10.5839,"has_fulltext":false,"cited_by_count":38,"citation_normalized_percentile":{"value":0.99337233,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"540","last_page":"553"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7929515838623047},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.7549594640731812},{"id":"https://openalex.org/keywords/memory-bandwidth","display_name":"Memory bandwidth","score":0.7201544046401978},{"id":"https://openalex.org/keywords/bandwidth","display_name":"Bandwidth (computing)","score":0.6918553113937378},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.6176475286483765},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4929032325744629},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.46209967136383057},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3562765121459961},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.15967383980751038},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.10337191820144653}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7929515838623047},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.7549594640731812},{"id":"https://openalex.org/C188045654","wikidata":"https://www.wikidata.org/wiki/Q17148339","display_name":"Memory bandwidth","level":2,"score":0.7201544046401978},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.6918553113937378},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.6176475286483765},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4929032325744629},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.46209967136383057},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3562765121459961},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.15967383980751038},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.10337191820144653}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/isca52012.2021.00049","is_oa":false,"landing_page_url":"https://doi.org/10.1109/isca52012.2021.00049","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 ACM/IEEE 48th Annual International Symposium on Computer Architecture (ISCA)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2007.00156","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2007.00156","pdf_url":"https://arxiv.org/pdf/2007.00156","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2007.00156","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2007.00156","pdf_url":"https://arxiv.org/pdf/2007.00156","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309480","display_name":"Nvidia","ror":"https://ror.org/03jdj4y14"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":61,"referenced_works":["https://openalex.org/W1515364274","https://openalex.org/W1539327817","https://openalex.org/W1929619420","https://openalex.org/W1962931680","https://openalex.org/W1980620372","https://openalex.org/W2009571103","https://openalex.org/W2042098886","https://openalex.org/W2057332538","https://openalex.org/W2108598243","https://openalex.org/W2109039688","https://openalex.org/W2131613942","https://openalex.org/W2157225945","https://openalex.org/W2194775991","https://openalex.org/W2287011250","https://openalex.org/W2525778437","https://openalex.org/W2606722458","https://openalex.org/W2625200202","https://openalex.org/W2755567635","https://openalex.org/W2790925711","https://openalex.org/W2796649226","https://openalex.org/W2898991608","https://openalex.org/W2900824371","https://openalex.org/W2903754802","https://openalex.org/W2904180680","https://openalex.org/W2905104204","https://openalex.org/W2922527104","https://openalex.org/W2926767350","https://openalex.org/W2947737663","https://openalex.org/W2949161920","https://openalex.org/W2973727699","https://openalex.org/W2979719709","https://openalex.org/W2980104813","https://openalex.org/W2980200167","https://openalex.org/W2989768222","https://openalex.org/W3010591261","https://openalex.org/W3016395792","https://openalex.org/W3036878841","https://openalex.org/W3040573126","https://openalex.org/W3043023836","https://openalex.org/W3043522163","https://openalex.org/W3081168214","https://openalex.org/W3085163101","https://openalex.org/W3096425133","https://openalex.org/W3097528158","https://openalex.org/W3156636320","https://openalex.org/W4244024631","https://openalex.org/W4297688475","https://openalex.org/W6687483927","https://openalex.org/W6696413332","https://openalex.org/W6750448596","https://openalex.org/W6756488470","https://openalex.org/W6760703128","https://openalex.org/W6761333114","https://openalex.org/W6763737044","https://openalex.org/W6764726765","https://openalex.org/W6767997687","https://openalex.org/W6770493259","https://openalex.org/W6775029814","https://openalex.org/W6779933952","https://openalex.org/W6780805062","https://openalex.org/W6795223348"],"related_works":["https://openalex.org/W2317245370","https://openalex.org/W4249323025","https://openalex.org/W198851386","https://openalex.org/W2030310580","https://openalex.org/W947442053","https://openalex.org/W1980160788","https://openalex.org/W2148915962","https://openalex.org/W2283866686","https://openalex.org/W4287182096","https://openalex.org/W27867058"],"abstract_inverted_index":{"Deep":[0],"Learning":[1],"(DL)":[2],"training":[3],"platforms":[4],"are":[5],"built":[6],"by":[7,133,164],"interconnecting":[8],"multiple":[9],"DL":[10,55,83,92,123,149],"accelerators":[11],"(e.g.,":[12],"GPU/TPU)":[13],"via":[14,65],"fast,":[15],"customized":[16],"interconnects":[17],"with":[18],"100s":[19],"of":[20,23,76,173],"gigabytes":[21],"(GBs)":[22],"bandwidth.":[24],"However,":[25],"as":[26],"we":[27,72,88],"identify":[28],"in":[29,126,170,188],"this":[30,33],"work,":[31],"driving":[32],"bandwidth":[34,80,162],"is":[35,39,42],"quite":[36],"challenging.":[37],"This":[38],"because":[40],"there":[41],"a":[43,90],"pernicious":[44],"balance":[45],"between":[46],"using":[47],"the":[48,104,110,116,129,139,159,199],"accelerator\u2019s":[49],"compute":[50,77,84,105,118],"and":[51,57,69,78,85,106,119,151,182,194],"memory":[52,79,120,131],"for":[53,82,122,191],"both":[54],"computations":[56],"communication.This":[58],"work":[59],"makes":[60],"two":[61],"key":[62],"contributions.":[63],"First,":[64],"real":[66],"system":[67],"measurements":[68],"detailed":[70],"modeling,":[71],"provide":[73],"an":[74,171],"understanding":[75],"demands":[81],"comms.":[86],"Second,":[87],"propose":[89],"novel":[91],"collective":[93],"communication":[94],"accelerator":[95,111],"called":[96],"Accelerator":[97],"Collectives":[98],"Engine":[99],"(ACE)":[100],"that":[101],"sits":[102],"alongside":[103],"networking":[107],"engines":[108],"at":[109],"endpoint.":[112],"ACE":[113],"frees":[114],"up":[115],"endpoint\u2019s":[117],"resources":[121],"compute,":[124],"which":[125],"turn":[127],"reduces":[128],"required":[130],"BW":[132,142],"3.5\u00d7":[134],"on":[135,156],"average":[136,172],"to":[137,144,167,176,180,185,198],"drive":[138],"same":[140],"network":[141,153,161],"compared":[143,197],"state-of-the-art":[145],"baselines.":[146],"For":[147],"modern":[148],"workloads":[150],"different":[152],"sizes,":[154],"ACE,":[155],"average,":[157],"increases":[158],"effective":[160],"utilization":[163],"1.44\u00d7":[165],"(up":[166,175,179,184],"2.67\u00d7),":[168],"resulting":[169],"1.41\u00d7":[174],"1.51\u00d7),":[177],"1.12\u00d7":[178],"1.17\u00d7),":[181],"1.13\u00d7":[183],"1.19\u00d7)":[186],"speedup":[187],"iteration":[189],"time":[190],"ResNet-50,":[192],"GNMT":[193],"DLRM":[195],"when":[196],"best":[200],"baseline":[201],"configuration,":[202],"respectively.":[203]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":12},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":1}],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2021-08-16T00:00:00"}