{"id":"https://openalex.org/W4327909439","doi":"https://doi.org/10.1109/tpds.2023.3247883","title":"Fold3D: Rethinking and Parallelizing Computational and Communicational Tasks in the Training of Large DNN Models","display_name":"Fold3D: Rethinking and Parallelizing Computational and Communicational Tasks in the Training of Large DNN Models","publication_year":2023,"publication_date":"2023-03-20","ids":{"openalex":"https://openalex.org/W4327909439","doi":"https://doi.org/10.1109/tpds.2023.3247883"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2023.3247883","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tpds.2023.3247883","pdf_url":"https://ieeexplore.ieee.org/ielx7/71/10075651/10050126.pdf","source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://ieeexplore.ieee.org/ielx7/71/10075651/10050126.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016904555","display_name":"Fanxin Li","orcid":"https://orcid.org/0000-0003-2268-3036"},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Fanxin Li","raw_affiliation_strings":["Department of Computer Science, The University of Hong Kong, Hong Kong, SAR, China"],"raw_orcid":"https://orcid.org/0000-0003-2268-3036","affiliations":[{"raw_affiliation_string":"Department of Computer Science, The University of Hong Kong, Hong Kong, SAR, China","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088629918","display_name":"Shixiong Zhao","orcid":"https://orcid.org/0000-0002-1643-2583"},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shixiong Zhao","raw_affiliation_strings":["Department of Computer Science, The University of Hong Kong, Hong Kong, SAR, China"],"raw_orcid":"https://orcid.org/0000-0002-1643-2583","affiliations":[{"raw_affiliation_string":"Department of Computer Science, The University of Hong Kong, Hong Kong, SAR, China","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072210108","display_name":"Yuhao Qing","orcid":null},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yuhao Qing","raw_affiliation_strings":["Department of Computer Science, The University of Hong Kong, Hong Kong, SAR, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, The University of Hong Kong, Hong Kong, SAR, China","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101527007","display_name":"Xusheng Chen","orcid":"https://orcid.org/0000-0002-2807-9780"},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xusheng Chen","raw_affiliation_strings":["Department of Computer Science, The University of Hong Kong, Hong Kong, SAR, China"],"raw_orcid":"https://orcid.org/0000-0002-2807-9780","affiliations":[{"raw_affiliation_string":"Department of Computer Science, The University of Hong Kong, Hong Kong, SAR, China","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035241109","display_name":"Xiuxian Guan","orcid":"https://orcid.org/0000-0001-6133-8388"},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xiuxian Guan","raw_affiliation_strings":["Department of Computer Science, The University of Hong Kong, Hong Kong, SAR, China"],"raw_orcid":"https://orcid.org/0000-0001-6133-8388","affiliations":[{"raw_affiliation_string":"Department of Computer Science, The University of Hong Kong, Hong Kong, SAR, China","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100350738","display_name":"Sen Wang","orcid":"https://orcid.org/0000-0002-1808-5239"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sen Wang","raw_affiliation_strings":["Theory Lab, 2012 Labs, Huawei Technoloies, Co. Ltd, Hong Kong, SAR, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Theory Lab, 2012 Labs, Huawei Technoloies, Co. Ltd, Hong Kong, SAR, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100368312","display_name":"Gong Zhang","orcid":"https://orcid.org/0000-0003-0283-7050"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Gong Zhang","raw_affiliation_strings":["Theory Lab, 2012 Labs, Huawei Technoloies, Co. Ltd, Hong Kong, SAR, China"],"raw_orcid":"https://orcid.org/0000-0003-0283-7050","affiliations":[{"raw_affiliation_string":"Theory Lab, 2012 Labs, Huawei Technoloies, Co. Ltd, Hong Kong, SAR, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015360935","display_name":"Heming Cui","orcid":"https://orcid.org/0000-0001-7746-440X"},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Heming Cui","raw_affiliation_strings":["Department of Computer Science, The University of Hong Kong, Hong Kong, SAR, China","Shanghai AI Lab, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0001-7746-440X","affiliations":[{"raw_affiliation_string":"Department of Computer Science, The University of Hong Kong, Hong Kong, SAR, China","institution_ids":["https://openalex.org/I889458895"]},{"raw_affiliation_string":"Shanghai AI Lab, Shanghai, China","institution_ids":["https://openalex.org/I4391012619"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.3182,"has_fulltext":true,"cited_by_count":14,"citation_normalized_percentile":{"value":0.79608776,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"34","issue":"5","first_page":"1432","last_page":"1449"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9934999942779541,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8198826313018799},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.6624811291694641},{"id":"https://openalex.org/keywords/data-parallelism","display_name":"Data parallelism","score":0.5348894000053406},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5317522287368774},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.46330079436302185},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.4589866101741791},{"id":"https://openalex.org/keywords/dimension","display_name":"Dimension (graph theory)","score":0.44608229398727417},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.40897876024246216},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.13640925288200378}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8198826313018799},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.6624811291694641},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.5348894000053406},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5317522287368774},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.46330079436302185},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.4589866101741791},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.44608229398727417},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40897876024246216},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.13640925288200378},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2023.3247883","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tpds.2023.3247883","pdf_url":"https://ieeexplore.ieee.org/ielx7/71/10075651/10050126.pdf","source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1109/tpds.2023.3247883","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tpds.2023.3247883","pdf_url":"https://ieeexplore.ieee.org/ielx7/71/10075651/10050126.pdf","source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4327909439.pdf","grobid_xml":"https://content.openalex.org/works/W4327909439.grobid-xml"},"referenced_works_count":89,"referenced_works":["https://openalex.org/W569156507","https://openalex.org/W2001905757","https://openalex.org/W2045271686","https://openalex.org/W2057332538","https://openalex.org/W2060393849","https://openalex.org/W2338908902","https://openalex.org/W2523246573","https://openalex.org/W2612387305","https://openalex.org/W2791186466","https://openalex.org/W2896457183","https://openalex.org/W2903697572","https://openalex.org/W2969388332","https://openalex.org/W2970155754","https://openalex.org/W2973727699","https://openalex.org/W2975712713","https://openalex.org/W2977038365","https://openalex.org/W2985738161","https://openalex.org/W2991040477","https://openalex.org/W3021234081","https://openalex.org/W3040573126","https://openalex.org/W3043840704","https://openalex.org/W3081168214","https://openalex.org/W3086105743","https://openalex.org/W3127806443","https://openalex.org/W3129831491","https://openalex.org/W3132107458","https://openalex.org/W3138516171","https://openalex.org/W3138895808","https://openalex.org/W3158631574","https://openalex.org/W3163465001","https://openalex.org/W3187018546","https://openalex.org/W3188065709","https://openalex.org/W3188270315","https://openalex.org/W3204998121","https://openalex.org/W3205803342","https://openalex.org/W3206832494","https://openalex.org/W4206706211","https://openalex.org/W4212774754","https://openalex.org/W4220703317","https://openalex.org/W4224308101","https://openalex.org/W4226479682","https://openalex.org/W4280611847","https://openalex.org/W4281922990","https://openalex.org/W4287391717","https://openalex.org/W4288089799","https://openalex.org/W4292779060","https://openalex.org/W4301239768","https://openalex.org/W4385245566","https://openalex.org/W6635935089","https://openalex.org/W6684249991","https://openalex.org/W6703652217","https://openalex.org/W6727249380","https://openalex.org/W6739901393","https://openalex.org/W6748645090","https://openalex.org/W6753584990","https://openalex.org/W6754818105","https://openalex.org/W6755207826","https://openalex.org/W6756379755","https://openalex.org/W6756718674","https://openalex.org/W6757053730","https://openalex.org/W6762122294","https://openalex.org/W6762211661","https://openalex.org/W6763701032","https://openalex.org/W6767997687","https://openalex.org/W6768142256","https://openalex.org/W6769062451","https://openalex.org/W6769275798","https://openalex.org/W6769627184","https://openalex.org/W6769865700","https://openalex.org/W6776277508","https://openalex.org/W6776671032","https://openalex.org/W6778883912","https://openalex.org/W6779863968","https://openalex.org/W6780239866","https://openalex.org/W6780805062","https://openalex.org/W6781728138","https://openalex.org/W6784333009","https://openalex.org/W6787953186","https://openalex.org/W6788811087","https://openalex.org/W6790235086","https://openalex.org/W6790665546","https://openalex.org/W6793032698","https://openalex.org/W6795581714","https://openalex.org/W6801859478","https://openalex.org/W6810081322","https://openalex.org/W6810475161","https://openalex.org/W6811928498","https://openalex.org/W6838632916","https://openalex.org/W6839827798"],"related_works":["https://openalex.org/W2950520577","https://openalex.org/W1554644772","https://openalex.org/W2494130044","https://openalex.org/W2003935582","https://openalex.org/W2033862586","https://openalex.org/W3209384898","https://openalex.org/W74409296","https://openalex.org/W1991844655","https://openalex.org/W1595834484","https://openalex.org/W3170887803"],"abstract_inverted_index":{"Training":[0],"a":[1,34,51,110],"large":[2],"DNN":[3,111,123],"(e.g.,":[4,20,73],"GPT3)":[5],"efficiently":[6],"on":[7],"commodity":[8,90],"clouds":[9],"is":[10],"challenging":[11],"even":[12],"with":[13,38,143],"the":[14,26,47,58,66,70,86,102,117,121,130,160],"latest":[15],"3D":[16,106],"parallel":[17],"training":[18,71,87,174],"systems":[19],"Megatron":[21,166],"v3.0).":[22],"In":[23],"particular,":[24],"along":[25,46],"pipeline":[27],"parallelism":[28,49,77],"dimension,":[29,50],"computational":[30,62,98,118,146],"tasks":[31,119,132],"that":[32,116,133],"produce":[33],"whole":[35],"DNN's":[36],"gradients":[37],"multiple":[39,113],"input":[40],"batches":[41],"should":[42],"be":[43,126,138],"concurrently":[44],"activated;":[45],"data":[48,76,169],"set":[52],"of":[53,61,85,159],"heavy-weight":[54],"communications":[55],"(for":[56,105],"aggregating":[57],"accumulated":[59],"outputs":[60],"tasks)":[63],"isinevitably":[64],"serializedafter":[65],"pipelined":[67],"tasks,":[68,99],"undermining":[69],"performance":[72,186],"in":[74,152,165],"Megatron,":[75],"caused":[78],"all":[79],"GPUs":[80],"idle":[81,163],"for":[82],"over":[83,89,182],"44%":[84,162],"time)":[88],"cloud":[91],"networks.":[92],"To":[93],"deserialize":[94],"these":[95],"communicational":[96,131],"and":[97,129,140],"we":[100],"propose":[101],"AIAO":[103],"scheduling":[104],"parallelism)":[107],"which":[108],"slices":[109],"into":[112],"segments,":[114],"so":[115],"processing":[120],"same":[122],"segment":[124,136],"can":[125,137],"scheduled":[127],"together,":[128],"synchronize":[134],"this":[135,150],"launched":[139],"overlapped":[141],"(deserialized)":[142],"other":[144],"segments\u2019":[145],"tasks.":[147],"We":[148],"realized":[149],"idea":[151],"ourFold3Dtraining":[153],"system.":[154],"Extensive":[155],"evaluation":[156],"showsFold3Deliminated":[157],"most":[158],"all-GPU":[161],"time":[164],"(caused":[167],"by":[168],"parallelism),":[170],"leading":[171],"to":[172,178,188],"25.2%\u201342.1%":[173],"throughput":[175],"improvement":[176],"compared":[177],"four":[179],"notable":[180],"baselines":[181],"various":[183],"settings;Fold3D's":[184],"high":[185],"scaled":[187],"many":[189],"GPUs.":[190]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":1}],"updated_date":"2026-08-12T07:12:00.856984","created_date":"2025-10-10T00:00:00"}