{"id":"https://openalex.org/W4399073645","doi":"https://doi.org/10.1109/tpds.2024.3406420","title":"A Multidimensional Communication Scheduling Method for Hybrid Parallel DNN Training","display_name":"A Multidimensional Communication Scheduling Method for Hybrid Parallel DNN Training","publication_year":2024,"publication_date":"2024-05-28","ids":{"openalex":"https://openalex.org/W4399073645","doi":"https://doi.org/10.1109/tpds.2024.3406420"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2024.3406420","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2024.3406420","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5106623069","display_name":"Shengwei Li","orcid":"https://orcid.org/0000-0002-7419-1511"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shengwei Li","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology, Changsha, China","National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology in Changsha, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0002-7419-1511","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology in Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100625354","display_name":"Kai L\u00fc","orcid":"https://orcid.org/0000-0002-6378-7002"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kai Lu","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology, Changsha, China","National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology in Changsha, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0002-6378-7002","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology in Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055426388","display_name":"Zhiquan Lai","orcid":"https://orcid.org/0000-0002-3458-4732"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiquan Lai","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology, Changsha, China","National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology in Changsha, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0002-3458-4732","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology in Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106623049","display_name":"Weijie Liu","orcid":"https://orcid.org/0000-0002-0354-9467"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weijie Liu","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology, Changsha, China","National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology in Changsha, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0002-0354-9467","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology in Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074708780","display_name":"Keshi Ge","orcid":"https://orcid.org/0000-0002-0669-6892"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Keshi Ge","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology, Changsha, China","National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology in Changsha, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0002-0669-6892","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology in Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100440903","display_name":"Dongsheng Li","orcid":"https://orcid.org/0000-0001-9743-2034"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongsheng Li","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology, Changsha, China","National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology in Changsha, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0001-9743-2034","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer, National University of Defense Technology in Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I170215575"],"apc_list":null,"apc_paid":null,"fwci":2.8612,"has_fulltext":false,"cited_by_count":14,"citation_normalized_percentile":{"value":0.9182819,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"35","issue":"8","first_page":"1415","last_page":"1428"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.785011887550354},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.5539448261260986},{"id":"https://openalex.org/keywords/asynchronous-communication","display_name":"Asynchronous communication","score":0.5210767984390259},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.46657881140708923},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.4207955300807953},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.12596115469932556}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.785011887550354},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.5539448261260986},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.5210767984390259},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46657881140708923},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.4207955300807953},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.12596115469932556},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2024.3406420","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2024.3406420","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3606055544","display_name":null,"funder_award_id":"62025208","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8818319823","display_name":null,"funder_award_id":"62302512","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":71,"referenced_works":["https://openalex.org/W1566289585","https://openalex.org/W2057332538","https://openalex.org/W2130531694","https://openalex.org/W2257408573","https://openalex.org/W2259472270","https://openalex.org/W2512924740","https://openalex.org/W2525778437","https://openalex.org/W2611669587","https://openalex.org/W2896457183","https://openalex.org/W2906007643","https://openalex.org/W2963748441","https://openalex.org/W2969388332","https://openalex.org/W2973727699","https://openalex.org/W2975712713","https://openalex.org/W3047537431","https://openalex.org/W3081168214","https://openalex.org/W3086105743","https://openalex.org/W3116059414","https://openalex.org/W3124352525","https://openalex.org/W3129831491","https://openalex.org/W3132107458","https://openalex.org/W3138516171","https://openalex.org/W3189259198","https://openalex.org/W3202134982","https://openalex.org/W3205803342","https://openalex.org/W3206832494","https://openalex.org/W4220741164","https://openalex.org/W4221167110","https://openalex.org/W4226057994","https://openalex.org/W4281790033","https://openalex.org/W4283207077","https://openalex.org/W4284974526","https://openalex.org/W4288089799","https://openalex.org/W4301239768","https://openalex.org/W4312681979","https://openalex.org/W4312910503","https://openalex.org/W4321487996","https://openalex.org/W4327909439","https://openalex.org/W4360831831","https://openalex.org/W4381551031","https://openalex.org/W4385245566","https://openalex.org/W4386709668","https://openalex.org/W4401352466","https://openalex.org/W6629028937","https://openalex.org/W6692563993","https://openalex.org/W6727690538","https://openalex.org/W6739693220","https://openalex.org/W6748645090","https://openalex.org/W6754818105","https://openalex.org/W6755207826","https://openalex.org/W6756718674","https://openalex.org/W6760703128","https://openalex.org/W6762211661","https://openalex.org/W6766978945","https://openalex.org/W6767997687","https://openalex.org/W6769475105","https://openalex.org/W6769627184","https://openalex.org/W6770052818","https://openalex.org/W6774934307","https://openalex.org/W6778729859","https://openalex.org/W6778883912","https://openalex.org/W6779863968","https://openalex.org/W6784333009","https://openalex.org/W6784425352","https://openalex.org/W6787953186","https://openalex.org/W6797242466","https://openalex.org/W6799372109","https://openalex.org/W6810296985","https://openalex.org/W6839700459","https://openalex.org/W6853287796","https://openalex.org/W6856663864"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2116677773","https://openalex.org/W2155261584","https://openalex.org/W2584231425","https://openalex.org/W2150611273","https://openalex.org/W2390279801","https://openalex.org/W4207086172","https://openalex.org/W2358668433","https://openalex.org/W2042919702"],"abstract_inverted_index":{"The":[0,41,50],"transformer-based":[1],"deep":[2],"neural":[3],"network":[4,176],"(DNN)":[5],"models":[6],"have":[7,179],"shown":[8],"considerable":[9],"success":[10],"across":[11],"diverse":[12],"tasks,":[13],"prompting":[14],"widespread":[15],"adoption":[16],"of":[17,57,78,103,166,183],"distributed":[18],"training":[19,35,102,218],"methods":[20],"such":[21],"as":[22],"data":[23,73,108,128],"parallelism":[24,106,217],"and":[25,107,132,140,174,190,192,221],"pipeline":[26,105,216],"parallelism.":[27,109],"With":[28],"the":[29,47,55,76,164,214,226],"increasing":[30],"parameter":[31],"number,":[32],"hybrid":[33,79,100,144],"parallel":[34,80,101,145],"becomes":[36],"imperative":[37],"to":[38,209,213,225],"scale":[39],"training.":[40,81,146],"primary":[42],"bottleneck":[43],"in":[44,65,99,143],"scaling":[45],"remains":[46],"communication":[48,51,58,93,122,131,142,150,167,172],"overhead.":[49],"scheduling":[52,94,135],"technique,":[53],"emphasizing":[54],"overlap":[56],"with":[59,196],"computation,":[60],"has":[61],"demonstrated":[62],"its":[63],"benefits":[64],"scaling.":[66],"However,":[67],"most":[68],"existing":[69],"works":[70],"focus":[71],"on":[72,188],"parallelism,":[74],"overlooking":[75],"nuances":[77],"In":[82],"this":[83],"paper,":[84],"we":[85],"propose":[86],"<italic":[87,115,119,152,158,184,204],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[88,116,120,153,159,185,205],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">TriRace</i>":[89,117,154,186,206],",":[90,124],"an":[91],"efficient":[92],"framework":[95],"for":[96],"accelerating":[97],"communications":[98],"asynchronous":[104],"To":[110,147],"achieve":[111],"effective":[112],"computation-communication":[113],"overlap,":[114],"introduces":[118],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">3D":[121],"scheduling</i>":[123],"which":[125,162],"adeptly":[126],"leverages":[127],"dependencies":[129],"between":[130],"computations,":[133],"efficiently":[134],"AllReduce":[136],"communication,":[137,139],"sparse":[138],"peer-to-peer":[141],"avoid":[148],"possible":[149],"contentions,":[151],"also":[155],"incorporates":[156],"a":[157,181],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">topology-aware":[160],"runtime</i>":[161],"optimizes":[163],"execution":[165],"operations":[168,173],"by":[169],"considering":[170],"ongoing":[171],"real-time":[175],"status.":[177],"We":[178],"implemented":[180],"prototype":[182],"based":[187],"PyTorch":[189],"Pipedream-2BW,":[191,220],"conducted":[193],"comprehensive":[194],"evaluations":[195],"three":[197],"representative":[198],"baselines.":[199],"Experimental":[200],"results":[201],"show":[202],"that":[203],"achieves":[207],"up":[208],"1.07\u20131.45\u00d7":[210],"speedup":[211,223],"compared":[212,224],"state-of-the-art":[215],"baseline":[219],"1.24\u20131.81\u00d7":[222],"Megatron.":[227]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}