{"id":"https://openalex.org/W3154431492","doi":"https://doi.org/10.1109/infocom42981.2021.9488678","title":"Near-Optimal Topology-adaptive Parameter Synchronization in Distributed DNN Training","display_name":"Near-Optimal Topology-adaptive Parameter Synchronization in Distributed DNN Training","publication_year":2021,"publication_date":"2021-05-10","ids":{"openalex":"https://openalex.org/W3154431492","doi":"https://doi.org/10.1109/infocom42981.2021.9488678","mag":"3154431492"},"language":"en","primary_location":{"id":"doi:10.1109/infocom42981.2021.9488678","is_oa":false,"landing_page_url":"https://doi.org/10.1109/infocom42981.2021.9488678","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE INFOCOM 2021 - IEEE Conference on Computer Communications","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100443041","display_name":"Zhe Zhang","orcid":"https://orcid.org/0000-0003-3057-9490"},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhe Zhang","raw_affiliation_strings":["The University of Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Hong Kong","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012597518","display_name":"Chuan Wu","orcid":"https://orcid.org/0000-0002-3144-4398"},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Chuan Wu","raw_affiliation_strings":["The University of Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Hong Kong","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066247159","display_name":"Zongpeng Li","orcid":"https://orcid.org/0000-0001-5351-2075"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zongpeng Li","raw_affiliation_strings":["Wuhan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Wuhan University","institution_ids":["https://openalex.org/I37461747"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.3188,"has_fulltext":false,"cited_by_count":14,"citation_normalized_percentile":{"value":0.83100589,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7844725847244263},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.7061538100242615},{"id":"https://openalex.org/keywords/network-topology","display_name":"Network topology","score":0.6522995829582214},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.6202878952026367},{"id":"https://openalex.org/keywords/partition","display_name":"Partition (number theory)","score":0.5209159255027771},{"id":"https://openalex.org/keywords/bandwidth","display_name":"Bandwidth (computing)","score":0.48617690801620483},{"id":"https://openalex.org/keywords/schedule","display_name":"Schedule","score":0.47451251745224},{"id":"https://openalex.org/keywords/asynchronous-communication","display_name":"Asynchronous communication","score":0.4585442841053009},{"id":"https://openalex.org/keywords/topology","display_name":"Topology (electrical circuits)","score":0.4583423137664795},{"id":"https://openalex.org/keywords/broadcasting","display_name":"Broadcasting (networking)","score":0.43702417612075806},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.29182568192481995},{"id":"https://openalex.org/keywords/channel","display_name":"Channel (broadcasting)","score":0.1291799247264862},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.11861759424209595}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7844725847244263},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.7061538100242615},{"id":"https://openalex.org/C199845137","wikidata":"https://www.wikidata.org/wiki/Q145490","display_name":"Network topology","level":2,"score":0.6522995829582214},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.6202878952026367},{"id":"https://openalex.org/C42812","wikidata":"https://www.wikidata.org/wiki/Q1082910","display_name":"Partition (number theory)","level":2,"score":0.5209159255027771},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.48617690801620483},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.47451251745224},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.4585442841053009},{"id":"https://openalex.org/C184720557","wikidata":"https://www.wikidata.org/wiki/Q7825049","display_name":"Topology (electrical circuits)","level":2,"score":0.4583423137664795},{"id":"https://openalex.org/C110157686","wikidata":"https://www.wikidata.org/wiki/Q922122","display_name":"Broadcasting (networking)","level":2,"score":0.43702417612075806},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.29182568192481995},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.1291799247264862},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11861759424209595},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/infocom42981.2021.9488678","is_oa":false,"landing_page_url":"https://doi.org/10.1109/infocom42981.2021.9488678","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE INFOCOM 2021 - IEEE Conference on Computer Communications","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W1967786783","https://openalex.org/W2057332538","https://openalex.org/W2060393849","https://openalex.org/W2083842231","https://openalex.org/W2096212808","https://openalex.org/W2126210439","https://openalex.org/W2130531694","https://openalex.org/W2143065961","https://openalex.org/W2154295079","https://openalex.org/W2159128662","https://openalex.org/W2160054705","https://openalex.org/W2194775991","https://openalex.org/W2265775419","https://openalex.org/W2518649733","https://openalex.org/W2787998955","https://openalex.org/W2884711234","https://openalex.org/W2892234294","https://openalex.org/W2896457183","https://openalex.org/W2900824371","https://openalex.org/W2901541570","https://openalex.org/W2926767350","https://openalex.org/W2946076799","https://openalex.org/W2962736495","https://openalex.org/W2962863496","https://openalex.org/W2963341956","https://openalex.org/W2969388332","https://openalex.org/W2975712713","https://openalex.org/W2980047166","https://openalex.org/W2981114289","https://openalex.org/W3011751313","https://openalex.org/W3038057116","https://openalex.org/W3038098022","https://openalex.org/W3081331745","https://openalex.org/W3101708369","https://openalex.org/W4238465620","https://openalex.org/W4256524787","https://openalex.org/W4288093768","https://openalex.org/W4301239768","https://openalex.org/W4306252072","https://openalex.org/W6642156526","https://openalex.org/W6665801690","https://openalex.org/W6745458143","https://openalex.org/W6748645090","https://openalex.org/W6753209298","https://openalex.org/W6754645438","https://openalex.org/W6755207826","https://openalex.org/W6756439839","https://openalex.org/W6756488470","https://openalex.org/W6769475105","https://openalex.org/W6780667297"],"related_works":["https://openalex.org/W2116677773","https://openalex.org/W2155261584","https://openalex.org/W2584231425","https://openalex.org/W2150611273","https://openalex.org/W4207086172","https://openalex.org/W2042919702","https://openalex.org/W4225981436","https://openalex.org/W2156185805","https://openalex.org/W2770353918","https://openalex.org/W2122502560"],"abstract_inverted_index":{"Distributed":[0],"machine":[1],"learning":[2],"with":[3,77,101,137,194],"multiple":[4],"concurrent":[5,102],"workers":[6,31,85,115,139],"has":[7],"been":[8],"widely":[9],"adopted":[10],"to":[11,64,93,156,167,191],"train":[12],"large":[13],"deep":[14],"neural":[15],"networks":[16],"(DNNs).":[17],"Parameter":[18,47],"synchronization":[19,48,110,130,187],"is":[20,89],"a":[21,51,147],"key":[22],"component":[23],"in":[24,86,120],"each":[25,142],"iteration":[26],"of":[27,54,135,149,175],"distributed":[28,121],"training,":[29],"where":[30],"exchange":[32],"locally":[33],"computed":[34],"gradients":[35],"through":[36],"an":[37],"AllReduce":[38,70],"operation":[39],"or":[40,71,145],"parameter":[41,45,109,129,186],"servers,":[42],"for":[43,116,143],"global":[44],"updates.":[46],"often":[49,90],"constitutes":[50],"significant":[52],"portion":[53],"the":[55,59,107,127,176],"training":[56,66],"time;":[57],"minimizing":[58],"communication":[60,119,169],"time":[61],"contributes":[62],"substantially":[63],"DNN":[65,122],"speed-up.":[67],"Standard":[68],"ring-based":[69],"PS":[72],"architecture":[73],"work":[74,105],"efficiently":[75],"mostly":[76],"homogeneous":[78],"inter-worker":[79],"connectivity.":[80],"However,":[81],"available":[82,159],"bandwidth":[83,160],"among":[84,114],"real-world":[87],"clusters":[88],"heterogeneous,":[91],"due":[92],"different":[94,138],"hardware":[95],"configurations,":[96],"switching":[97],"topologies,":[98],"and":[99,112,161,164,179],"contention":[100],"jobs.":[103],"This":[104],"investigates":[106],"best":[108],"topology":[111,131],"schedule":[113],"most":[117],"expedited":[118],"training.":[123],"We":[124,151,171],"show":[125,180],"that":[126,181],"optimal":[128],"should":[132],"be":[133],"comprised":[134],"trees":[136],"as":[140],"roots,":[141],"aggregating":[144],"broadcasting":[146],"partition":[148],"gradients/parameters.":[150],"identify":[152],"near-optimal":[153],"forest":[154],"packing":[155],"maximally":[157],"utilize":[158],"overlap":[162],"aggregation":[163],"broadcast":[165],"stages":[166],"minimize":[168],"time.":[170],"provide":[172],"theoretical":[173],"analysis":[174],"performance":[177],"bound,":[178],"our":[182],"scheme":[183],"outperforms":[184],"state-of-the-art":[185],"schemes":[188],"by":[189],"up":[190],"18.3":[192],"times":[193],"extensive":[195],"evaluation":[196],"under":[197],"various":[198],"settings.":[199]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}