{"id":"https://openalex.org/W4387490420","doi":"https://doi.org/10.1109/tpds.2023.3323282","title":"Communication Optimization Algorithms for Distributed Deep Learning Systems: A Survey","display_name":"Communication Optimization Algorithms for Distributed Deep Learning Systems: A Survey","publication_year":2023,"publication_date":"2023-10-10","ids":{"openalex":"https://openalex.org/W4387490420","doi":"https://doi.org/10.1109/tpds.2023.3323282"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2023.3323282","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2023.3323282","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5080680105","display_name":"Enda Yu","orcid":"https://orcid.org/0000-0003-2661-0889"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Enda Yu","raw_affiliation_strings":["College of Computer, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0003-2661-0889","affiliations":[{"raw_affiliation_string":"College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006729432","display_name":"Dezun Dong","orcid":"https://orcid.org/0000-0001-6243-8479"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dezun Dong","raw_affiliation_strings":["College of Computer, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0001-6243-8479","affiliations":[{"raw_affiliation_string":"College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5000688713","display_name":"Xiangke Liao","orcid":"https://orcid.org/0000-0002-6125-3330"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiangke Liao","raw_affiliation_strings":["College of Computer, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-6125-3330","affiliations":[{"raw_affiliation_string":"College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I170215575"],"apc_list":null,"apc_paid":null,"fwci":1.2555,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.79440537,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"34","issue":"12","first_page":"3294","last_page":"3308"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8527404069900513},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.713238537311554},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.5777043104171753},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.5540018081665039},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5162122845649719},{"id":"https://openalex.org/keywords/optimization-problem","display_name":"Optimization problem","score":0.47910669445991516},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4535350799560547},{"id":"https://openalex.org/keywords/communications-system","display_name":"Communications system","score":0.4506356120109558},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.4359913468360901},{"id":"https://openalex.org/keywords/reduction","display_name":"Reduction (mathematics)","score":0.4335160553455353},{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.4304395914077759},{"id":"https://openalex.org/keywords/distributed-learning","display_name":"Distributed learning","score":0.4115694761276245},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3917110860347748},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.13502612709999084}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8527404069900513},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.713238537311554},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5777043104171753},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.5540018081665039},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5162122845649719},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.47910669445991516},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4535350799560547},{"id":"https://openalex.org/C101765175","wikidata":"https://www.wikidata.org/wiki/Q577764","display_name":"Communications system","level":2,"score":0.4506356120109558},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4359913468360901},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.4335160553455353},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.4304395914077759},{"id":"https://openalex.org/C2779582901","wikidata":"https://www.wikidata.org/wiki/Q21013010","display_name":"Distributed learning","level":2,"score":0.4115694761276245},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3917110860347748},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.13502612709999084},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2023.3323282","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2023.3323282","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G8818319823","display_name":null,"funder_award_id":"62302512","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":192,"referenced_works":["https://openalex.org/W114517082","https://openalex.org/W1442374986","https://openalex.org/W1446909886","https://openalex.org/W1518479949","https://openalex.org/W1652793671","https://openalex.org/W1686810756","https://openalex.org/W1845277745","https://openalex.org/W1861492603","https://openalex.org/W2108598243","https://openalex.org/W2132737349","https://openalex.org/W2138243089","https://openalex.org/W2138302120","https://openalex.org/W2155503253","https://openalex.org/W2163605009","https://openalex.org/W2166706236","https://openalex.org/W2168231600","https://openalex.org/W2186615578","https://openalex.org/W2194775991","https://openalex.org/W2336650964","https://openalex.org/W2405578611","https://openalex.org/W2407022425","https://openalex.org/W2512971201","https://openalex.org/W2573285109","https://openalex.org/W2606722458","https://openalex.org/W2606891064","https://openalex.org/W2617766261","https://openalex.org/W2622263826","https://openalex.org/W2626580042","https://openalex.org/W2741269719","https://openalex.org/W2744599870","https://openalex.org/W2745269232","https://openalex.org/W2766140019","https://openalex.org/W2766164908","https://openalex.org/W2769644379","https://openalex.org/W2769856846","https://openalex.org/W2774000609","https://openalex.org/W2775776326","https://openalex.org/W2789197829","https://openalex.org/W2805997383","https://openalex.org/W2807778630","https://openalex.org/W2886288284","https://openalex.org/W2888206291","https://openalex.org/W2891735383","https://openalex.org/W2896457183","https://openalex.org/W2900182564","https://openalex.org/W2900824371","https://openalex.org/W2901541570","https://openalex.org/W2904556356","https://openalex.org/W2911546317","https://openalex.org/W2914342100","https://openalex.org/W2926655273","https://openalex.org/W2942889782","https://openalex.org/W2946270031","https://openalex.org/W2946900926","https://openalex.org/W2950826569","https://openalex.org/W2953384591","https://openalex.org/W2962911728","https://openalex.org/W2963112712","https://openalex.org/W2963179579","https://openalex.org/W2963766684","https://openalex.org/W2963964896","https://openalex.org/W2964163156","https://openalex.org/W2966297898","https://openalex.org/W2967558351","https://openalex.org/W2969388332","https://openalex.org/W2970020383","https://openalex.org/W2972087877","https://openalex.org/W2973727699","https://openalex.org/W2973781261","https://openalex.org/W2975043678","https://openalex.org/W2975712713","https://openalex.org/W2978015420","https://openalex.org/W2982475424","https://openalex.org/W2985108934","https://openalex.org/W2988070836","https://openalex.org/W2990061623","https://openalex.org/W2991040477","https://openalex.org/W2995435108","https://openalex.org/W3000512942","https://openalex.org/W3004495293","https://openalex.org/W3012220622","https://openalex.org/W3015585426","https://openalex.org/W3015835112","https://openalex.org/W3016719260","https://openalex.org/W3030589255","https://openalex.org/W3032687999","https://openalex.org/W3035199578","https://openalex.org/W3037847693","https://openalex.org/W3038104246","https://openalex.org/W3047002910","https://openalex.org/W3047357290","https://openalex.org/W3081168214","https://openalex.org/W3090287616","https://openalex.org/W3095319910","https://openalex.org/W3099748883","https://openalex.org/W3101036738","https://openalex.org/W3102310167","https://openalex.org/W3102606411","https://openalex.org/W3102816259","https://openalex.org/W3124349893","https://openalex.org/W3124352525","https://openalex.org/W3132107458","https://openalex.org/W3132977829","https://openalex.org/W3134069806","https://openalex.org/W3154219392","https://openalex.org/W3166384310","https://openalex.org/W3170906335","https://openalex.org/W3176615255","https://openalex.org/W3177554119","https://openalex.org/W3193250980","https://openalex.org/W3202009823","https://openalex.org/W3204244704","https://openalex.org/W3205564238","https://openalex.org/W3206236031","https://openalex.org/W3206832494","https://openalex.org/W4221008227","https://openalex.org/W4224308101","https://openalex.org/W4281662083","https://openalex.org/W4283394632","https://openalex.org/W4288357791","https://openalex.org/W4295312788","https://openalex.org/W4297685247","https://openalex.org/W4299802343","https://openalex.org/W4301239768","https://openalex.org/W4364382637","https://openalex.org/W4381620519","https://openalex.org/W6628367914","https://openalex.org/W6628377381","https://openalex.org/W6631132026","https://openalex.org/W6636806777","https://openalex.org/W6637373629","https://openalex.org/W6638803421","https://openalex.org/W6639102338","https://openalex.org/W6679393576","https://openalex.org/W6680402377","https://openalex.org/W6680748266","https://openalex.org/W6684191040","https://openalex.org/W6684249991","https://openalex.org/W6684859321","https://openalex.org/W6686509673","https://openalex.org/W6703420464","https://openalex.org/W6713134421","https://openalex.org/W6731617708","https://openalex.org/W6738460352","https://openalex.org/W6739554518","https://openalex.org/W6739622702","https://openalex.org/W6741986022","https://openalex.org/W6745410505","https://openalex.org/W6745723224","https://openalex.org/W6746200960","https://openalex.org/W6746514494","https://openalex.org/W6746839373","https://openalex.org/W6748019269","https://openalex.org/W6748373249","https://openalex.org/W6748645090","https://openalex.org/W6751646691","https://openalex.org/W6752012617","https://openalex.org/W6752191696","https://openalex.org/W6753584990","https://openalex.org/W6753958361","https://openalex.org/W6754341472","https://openalex.org/W6754776447","https://openalex.org/W6754818105","https://openalex.org/W6754930508","https://openalex.org/W6755207826","https://openalex.org/W6756439839","https://openalex.org/W6756488470","https://openalex.org/W6756718674","https://openalex.org/W6756976665","https://openalex.org/W6758358981","https://openalex.org/W6758647027","https://openalex.org/W6760703128","https://openalex.org/W6761074043","https://openalex.org/W6762211661","https://openalex.org/W6762287338","https://openalex.org/W6762890684","https://openalex.org/W6763138067","https://openalex.org/W6766767561","https://openalex.org/W6766831760","https://openalex.org/W6766978945","https://openalex.org/W6767997687","https://openalex.org/W6768511045","https://openalex.org/W6770379276","https://openalex.org/W6775649616","https://openalex.org/W6778984096","https://openalex.org/W6779445818","https://openalex.org/W6780043775","https://openalex.org/W6789169260","https://openalex.org/W6796615672","https://openalex.org/W6802464851","https://openalex.org/W6810081322","https://openalex.org/W6852492453"],"related_works":["https://openalex.org/W2595172197","https://openalex.org/W2084856301","https://openalex.org/W2127970246","https://openalex.org/W2885125400","https://openalex.org/W1001352512","https://openalex.org/W1989889224","https://openalex.org/W4382618745","https://openalex.org/W1973775000","https://openalex.org/W2748922771","https://openalex.org/W1987128138"],"abstract_inverted_index":{"Deep":[0],"learning's":[1],"widespread":[2],"adoption":[3],"in":[4,30,74,110],"various":[5],"fields":[6],"has":[7],"made":[8],"distributed":[9,31,45,118],"training":[10,25],"across":[11],"multiple":[12],"computing":[13],"nodes":[14,20],"essential.":[15],"However,":[16],"frequent":[17],"communication":[18,41,60,91,93,95,114],"between":[19],"can":[21],"significantly":[22],"slow":[23],"down":[24],"speed,":[26],"creating":[27],"a":[28,54],"bottleneck":[29],"training.":[32],"To":[33],"address":[34],"this":[35,50],"issue,":[36],"researchers":[37],"are":[38],"focusing":[39],"on":[40,64,85],"optimization":[42,61,87,115],"algorithms":[43,62,116],"for":[44,117],"deep":[46,119],"learning":[47,120],"systems.":[48,121],"In":[49],"paper,":[51],"we":[52,102],"propose":[53],"standard":[55],"that":[56],"systematically":[57],"classifies":[58],"all":[59],"based":[63,84],"mathematical":[65],"modeling,":[66],"which":[67],"is":[68],"not":[69],"achieved":[70],"by":[71],"existing":[72,79],"surveys":[73],"the":[75,86,111],"field.":[76],"We":[77],"categorize":[78],"works":[80],"into":[81],"four":[82],"categories":[83],"strategies":[88],"of":[89,113],"communication:":[90],"masking,":[92],"compression,":[94],"frequency":[96],"reduction,":[97],"and":[98,107],"hybrid":[99],"optimization.":[100],"Finally,":[101],"discuss":[103],"potential":[104],"future":[105],"challenges":[106],"research":[108],"directions":[109],"field":[112]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":5}],"updated_date":"2026-08-08T01:25:22.217667","created_date":"2025-10-10T00:00:00"}