{"id":"https://openalex.org/W3193250980","doi":"https://doi.org/10.1109/infocom42981.2021.9488803","title":"Exploiting Simultaneous Communications to Accelerate Data Parallel Distributed Deep Learning","display_name":"Exploiting Simultaneous Communications to Accelerate Data Parallel Distributed Deep Learning","publication_year":2021,"publication_date":"2021-05-10","ids":{"openalex":"https://openalex.org/W3193250980","doi":"https://doi.org/10.1109/infocom42981.2021.9488803","mag":"3193250980"},"language":"en","primary_location":{"id":"doi:10.1109/infocom42981.2021.9488803","is_oa":false,"landing_page_url":"https://doi.org/10.1109/infocom42981.2021.9488803","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE INFOCOM 2021 - IEEE Conference on Computer Communications","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016836702","display_name":"Shaohuai Shi","orcid":"https://orcid.org/0000-0002-1418-5160"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":true,"raw_author_name":"Shaohuai Shi","raw_affiliation_strings":["The Hong Kong University of Science and Technology"],"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100730785","display_name":"Xiaowen Chu","orcid":"https://orcid.org/0000-0001-9745-4372"},"institutions":[{"id":"https://openalex.org/I141568987","display_name":"Hong Kong Baptist University","ror":"https://ror.org/0145fw131","country_code":"HK","type":"education","lineage":["https://openalex.org/I141568987"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xiaowen Chu","raw_affiliation_strings":["Hong Kong Baptist University"],"affiliations":[{"raw_affiliation_string":"Hong Kong Baptist University","institution_ids":["https://openalex.org/I141568987"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100374506","display_name":"Bo Li","orcid":"https://orcid.org/0000-0003-2083-9105"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Bo Li","raw_affiliation_strings":["The Hong Kong University of Science and Technology"],"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology","institution_ids":["https://openalex.org/I200769079"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5016836702"],"corresponding_institution_ids":["https://openalex.org/I200769079"],"apc_list":null,"apc_paid":null,"fwci":2.234,"has_fulltext":false,"cited_by_count":34,"citation_normalized_percentile":{"value":0.89897014,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8478846549987793},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.7817850112915039},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.6421307325363159},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.5247621536254883},{"id":"https://openalex.org/keywords/schedule","display_name":"Schedule","score":0.4966595768928528},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.4493109881877899},{"id":"https://openalex.org/keywords/tensor","display_name":"Tensor (intrinsic definition)","score":0.44167959690093994},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.43873023986816406},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3781435191631317},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.35307741165161133},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.29489392042160034},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.15187272429466248}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8478846549987793},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.7817850112915039},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.6421307325363159},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.5247621536254883},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.4966595768928528},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4493109881877899},{"id":"https://openalex.org/C155281189","wikidata":"https://www.wikidata.org/wiki/Q3518150","display_name":"Tensor (intrinsic definition)","level":2,"score":0.44167959690093994},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.43873023986816406},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3781435191631317},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.35307741165161133},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.29489392042160034},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.15187272429466248},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/infocom42981.2021.9488803","is_oa":false,"landing_page_url":"https://doi.org/10.1109/infocom42981.2021.9488803","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE INFOCOM 2021 - IEEE Conference on Computer Communications","raw_type":"proceedings-article"},{"id":"pmh:oai:repository.ust.hk:1783.1-112219","is_oa":false,"landing_page_url":"http://www.scopus.com/record/display.url?eid=2-s2.0-85111915788&origin=inward","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference paper"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":78,"referenced_works":["https://openalex.org/W778657980","https://openalex.org/W1572016165","https://openalex.org/W2132737349","https://openalex.org/W2160054705","https://openalex.org/W2168231600","https://openalex.org/W2194775991","https://openalex.org/W2410733619","https://openalex.org/W2535374105","https://openalex.org/W2580688187","https://openalex.org/W2617766261","https://openalex.org/W2626580042","https://openalex.org/W2745269232","https://openalex.org/W2766140019","https://openalex.org/W2769644379","https://openalex.org/W2787998955","https://openalex.org/W2884711234","https://openalex.org/W2896457183","https://openalex.org/W2901541570","https://openalex.org/W2911863041","https://openalex.org/W2919632853","https://openalex.org/W2920397365","https://openalex.org/W2920668770","https://openalex.org/W2950826569","https://openalex.org/W2962758826","https://openalex.org/W2963341956","https://openalex.org/W2963446712","https://openalex.org/W2963483475","https://openalex.org/W2963540381","https://openalex.org/W2963786636","https://openalex.org/W2963903325","https://openalex.org/W2964004663","https://openalex.org/W2964079725","https://openalex.org/W2964324519","https://openalex.org/W2964350391","https://openalex.org/W2966527647","https://openalex.org/W2967558351","https://openalex.org/W2975712713","https://openalex.org/W2980047166","https://openalex.org/W2982475424","https://openalex.org/W2985108934","https://openalex.org/W3037338728","https://openalex.org/W3037875189","https://openalex.org/W3038098022","https://openalex.org/W3039165326","https://openalex.org/W3047357290","https://openalex.org/W3047537431","https://openalex.org/W3094326709","https://openalex.org/W3099748883","https://openalex.org/W3105148963","https://openalex.org/W3124352525","https://openalex.org/W3138056726","https://openalex.org/W3154560182","https://openalex.org/W3177554119","https://openalex.org/W4288093768","https://openalex.org/W4288357791","https://openalex.org/W4301239768","https://openalex.org/W6622473587","https://openalex.org/W6679393576","https://openalex.org/W6684859321","https://openalex.org/W6714879114","https://openalex.org/W6738460352","https://openalex.org/W6739554518","https://openalex.org/W6739693220","https://openalex.org/W6745723224","https://openalex.org/W6746200960","https://openalex.org/W6748645090","https://openalex.org/W6753209298","https://openalex.org/W6754818105","https://openalex.org/W6755207826","https://openalex.org/W6756439839","https://openalex.org/W6758358981","https://openalex.org/W6759786419","https://openalex.org/W6762211661","https://openalex.org/W6766092415","https://openalex.org/W6769475105","https://openalex.org/W6780667297","https://openalex.org/W6784701502","https://openalex.org/W6786422851"],"related_works":["https://openalex.org/W2058965144","https://openalex.org/W2164382479","https://openalex.org/W98480971","https://openalex.org/W2150291671","https://openalex.org/W2027972911","https://openalex.org/W2146343568","https://openalex.org/W2013643406","https://openalex.org/W2157978810","https://openalex.org/W2966297898","https://openalex.org/W2955229517"],"abstract_inverted_index":{"Synchronous":[0],"stochastic":[1],"gradient":[2],"descent":[3],"(S-SGD)":[4],"with":[5,152,168],"data":[6],"parallelism":[7],"is":[8,34,59],"widely":[9],"used":[10],"for":[11,55],"training":[12,32,125,149],"deep":[13],"learning":[14],"(DL)":[15],"models":[16],"in":[17,128],"distributed":[18,148],"systems.":[19],"A":[20],"pipelined":[21,45],"schedule":[22,72],"of":[23,29,90,114,122,165],"the":[24,73,87,111,124,147,188,198],"computing":[25],"and":[26,100,133,145,154,193],"communication":[27,41,65,74,112],"tasks":[28,75],"a":[30,56,60],"DL":[31],"job":[33],"an":[35,119,140,161],"effective":[36],"scheme":[37],"to":[38,63],"hide":[39],"some":[40,51],"costs.":[42],"In":[43,82],"such":[44],"S-SGD,":[46],"tensor":[47,69,131,191,200],"fusion":[48,70,132,201],"(i.e.,":[49],"merging":[50],"consecutive":[52],"layers'":[53],"gradients":[54],"single":[57],"communication)":[58],"key":[61],"ingredient":[62],"improve":[64,110],"efficiency.":[66],"However,":[67],"existing":[68],"techniques":[71],"sequentially,":[76],"which":[77,129],"overlooks":[78],"their":[79],"independence":[80],"nature.":[81],"this":[83],"paper,":[84],"we":[85,102],"expand":[86],"design":[88],"space":[89],"scheduling":[91,143],"by":[92],"exploiting":[93],"simultaneous":[94,105,134],"All-Reduce":[95,106],"communications.":[96],"Through":[97],"theoretical":[98],"analysis":[99],"experiments,":[101],"show":[103,177],"that":[104,178],"communications":[107,135],"can":[108,180],"effectively":[109],"efficiency":[113],"small":[115],"tensors.":[116],"We":[117,138,156],"formulate":[118],"optimization":[120],"problem":[121],"minimizing":[123],"iteration":[126],"time,":[127],"both":[130],"are":[136],"allowed.":[137],"develop":[139],"efficient":[141],"optimal":[142],"solution":[144],"implement":[146],"algorithm":[150],"ASC-WFBP":[151,179],"Horovod":[153],"PyTorch.":[155],"conduct":[157],"real-world":[158],"experiments":[159],"on":[160,173],"8-node":[162],"GPU":[163],"cluster":[164],"32":[166],"GPUs":[167],"10Gbps":[169],"Ethernet.":[170],"Experimental":[171],"results":[172],"four":[174],"modern":[175],"DNNs":[176],"achieve":[181],"about":[182],"1.09":[183],"\u00d7":[184],"-2.48\u00d7":[185],"speedup":[186,196],"over":[187,197],"baseline":[189],"without":[190],"fusion,":[192],"1.15\u00d7":[194],"-1.35\u00d7":[195],"state-of-the-art":[199],"solution.":[202]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":13},{"year":2022,"cited_by_count":3}],"updated_date":"2026-04-02T15:55:50.835912","created_date":"2025-10-10T00:00:00"}