{"id":"https://openalex.org/W4396712969","doi":"https://doi.org/10.1109/tpds.2024.3397800","title":"AutoDDL: Automatic Distributed Deep Learning With Near-Optimal Bandwidth Cost","display_name":"AutoDDL: Automatic Distributed Deep Learning With Near-Optimal Bandwidth Cost","publication_year":2024,"publication_date":"2024-05-07","ids":{"openalex":"https://openalex.org/W4396712969","doi":"https://doi.org/10.1109/tpds.2024.3397800"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2024.3397800","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2024.3397800","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102713915","display_name":"Jinfan Chen","orcid":"https://orcid.org/0009-0006-4922-5165"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Jinfan Chen","raw_affiliation_strings":["Department of Computer Science, ETH Zurich, Z&#x00FC;rich, Switzerland"],"raw_orcid":"https://orcid.org/0009-0006-4922-5165","affiliations":[{"raw_affiliation_string":"Department of Computer Science, ETH Zurich, Z&#x00FC;rich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088172355","display_name":"Shigang Li","orcid":"https://orcid.org/0000-0003-0022-7865"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shigang Li","raw_affiliation_strings":["School of Computer Science, Beijing University of Posts and Telecommunications, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-0022-7865","affiliations":[{"raw_affiliation_string":"School of Computer Science, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Ran Guo","orcid":"https://orcid.org/0009-0003-6835-2467"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ran Guo","raw_affiliation_strings":["OneFlow, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0003-6835-2467","affiliations":[{"raw_affiliation_string":"OneFlow, Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101489641","display_name":"Yuan Jinhui","orcid":"https://orcid.org/0000-0002-0700-2645"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jinhui Yuan","raw_affiliation_strings":["OneFlow, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-0700-2645","affiliations":[{"raw_affiliation_string":"OneFlow, Beijing, China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5026990786","display_name":"Torsten Hoefler","orcid":"https://orcid.org/0000-0002-1333-9797"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Torsten Hoefler","raw_affiliation_strings":["Department of Computer Science, ETH Zurich, Z&#x00FC;rich, Switzerland"],"raw_orcid":"https://orcid.org/0000-0002-1333-9797","affiliations":[{"raw_affiliation_string":"Department of Computer Science, ETH Zurich, Z&#x00FC;rich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.3312,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.79763675,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"35","issue":"8","first_page":"1331","last_page":"1344"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8271473050117493},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5505627393722534},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.4976048767566681},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.4647696614265442},{"id":"https://openalex.org/keywords/bandwidth","display_name":"Bandwidth (computing)","score":0.44358137249946594},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.4124048948287964},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3514821529388428},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.34659677743911743},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.25541695952415466},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.16082125902175903},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.09659796953201294}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8271473050117493},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5505627393722534},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.4976048767566681},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.4647696614265442},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.44358137249946594},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.4124048948287964},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3514821529388428},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.34659677743911743},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.25541695952415466},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.16082125902175903},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.09659796953201294}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2024.3397800","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2024.3397800","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1437459877","display_name":null,"funder_award_id":"62372055","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":75,"referenced_works":["https://openalex.org/W201315547","https://openalex.org/W1598866093","https://openalex.org/W1598993252","https://openalex.org/W1686810756","https://openalex.org/W2000882268","https://openalex.org/W2010747199","https://openalex.org/W2012652661","https://openalex.org/W2057332538","https://openalex.org/W2065231486","https://openalex.org/W2083842231","https://openalex.org/W2131613942","https://openalex.org/W2135194391","https://openalex.org/W2138309709","https://openalex.org/W2168748809","https://openalex.org/W2338908902","https://openalex.org/W2525778437","https://openalex.org/W2883830791","https://openalex.org/W2947737663","https://openalex.org/W2966527647","https://openalex.org/W2967558351","https://openalex.org/W2969388332","https://openalex.org/W2972073717","https://openalex.org/W2972087877","https://openalex.org/W2982475424","https://openalex.org/W2983101505","https://openalex.org/W2984305089","https://openalex.org/W2991040477","https://openalex.org/W3001279689","https://openalex.org/W3091097978","https://openalex.org/W3099748883","https://openalex.org/W3101104221","https://openalex.org/W3111443264","https://openalex.org/W3130395060","https://openalex.org/W3132107458","https://openalex.org/W3132977829","https://openalex.org/W3159351344","https://openalex.org/W3166243953","https://openalex.org/W3167220634","https://openalex.org/W3206832494","https://openalex.org/W3209444800","https://openalex.org/W3210871626","https://openalex.org/W4224226923","https://openalex.org/W4288094025","https://openalex.org/W4294433898","https://openalex.org/W4297685247","https://openalex.org/W4301239768","https://openalex.org/W4319453176","https://openalex.org/W4321636758","https://openalex.org/W4385245566","https://openalex.org/W6635810480","https://openalex.org/W6637373629","https://openalex.org/W6684191040","https://openalex.org/W6703652217","https://openalex.org/W6727690538","https://openalex.org/W6746200960","https://openalex.org/W6747917562","https://openalex.org/W6748019269","https://openalex.org/W6748645090","https://openalex.org/W6753584990","https://openalex.org/W6763737044","https://openalex.org/W6769275798","https://openalex.org/W6772383348","https://openalex.org/W6778883912","https://openalex.org/W6779863968","https://openalex.org/W6780805062","https://openalex.org/W6790665546","https://openalex.org/W6793964378","https://openalex.org/W6795846692","https://openalex.org/W6796811805","https://openalex.org/W6797720797","https://openalex.org/W6803433826","https://openalex.org/W6804052296","https://openalex.org/W6811928498","https://openalex.org/W6847028765","https://openalex.org/W6849027614"],"related_works":["https://openalex.org/W3037187668","https://openalex.org/W4231775656","https://openalex.org/W2046435967","https://openalex.org/W4234772502","https://openalex.org/W2380685755","https://openalex.org/W2383646825","https://openalex.org/W2371018915","https://openalex.org/W2252100032","https://openalex.org/W2963436428","https://openalex.org/W2354191502"],"abstract_inverted_index":{"Recent":[0],"advances":[1],"in":[2],"deep":[3],"learning":[4,40],"are":[5],"driven":[6],"by":[7,73,139],"the":[8,66,109,130,135,156],"growing":[9],"scale":[10],"of":[11,28,70],"computation,":[12],"data,":[13,29],"and":[14,31,55,68,84,118,126,143,147,151],"models.":[15],"However,":[16],"efficiently":[17],"training":[18,50,137],"large-scale":[19],"models":[20],"on":[21,38,116,155],"distributed":[22,49],"systems":[23],"requires":[24],"an":[25,95],"intricate":[26],"combination":[27],"operator,":[30],"pipeline":[32],"parallelism,":[33],"which":[34,106],"exerts":[35],"heavy":[36],"burden":[37],"machine":[39],"practitioners.":[41],"To":[42],"this":[43],"end,":[44],"we":[45],"propose":[46],"AutoDDL,":[47],"a":[48,101],"framework":[51],"that":[52],"automatically":[53],"explores":[54],"exploits":[56],"new":[57],"parallelization":[58],"schemes":[59,72],"with":[60,94,100],"near-optimal":[61],"bandwidth":[62],"cost.":[63],"AutoDDL":[64,91,133],"facilitates":[65],"description":[67],"implementation":[69],"different":[71,122],"utilizing":[74],"OneFlow's":[75],"<italic":[76,80,85],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[77,81,86],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Split</i>":[78],",":[79,83],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Broadcast</i>":[82],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Partial":[87],"Sum</i>":[88],"(SBP)":[89],"abstraction.":[90],"is":[92],"equipped":[93],"analytical":[96],"performance":[97],"model":[98],"combined":[99],"customized":[102],"Coordinate":[103],"Descent":[104],"algorithm,":[105],"significantly":[107],"reduces":[108,134],"scheme":[110],"searching":[111],"overhead.":[112],"We":[113],"conduct":[114],"evaluations":[115],"Multi-Node-Single-GPU":[117],"Multi-Node-Multi-GPU":[119],"machines":[120],"using":[121],"models,":[123],"including":[124],"VGG":[125,154],"Transformer.":[127],"Compared":[128],"to":[129,141,149],"expert-optimized":[131],"implementations,":[132],"end-to-end":[136],"time":[138],"up":[140,148],"31.1%":[142],"10%":[144],"for":[145,153],"Transformer":[146],"17.7%":[150],"71.5%":[152],"two":[157],"parallel":[158],"systems,":[159],"respectively.":[160]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}