{"id":"https://openalex.org/W3019389960","doi":"https://doi.org/10.1109/tpds.2021.3132413","title":"TensorOpt: Exploring the Tradeoffs in Distributed DNN Training With Auto-Parallelism","display_name":"TensorOpt: Exploring the Tradeoffs in Distributed DNN Training With Auto-Parallelism","publication_year":2021,"publication_date":"2021-12-06","ids":{"openalex":"https://openalex.org/W3019389960","doi":"https://doi.org/10.1109/tpds.2021.3132413","mag":"3019389960"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2021.3132413","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2021.3132413","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2004.10856","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000220537","display_name":"Zhenkun Cai","orcid":"https://orcid.org/0000-0002-0199-4866"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhenkun Cai","raw_affiliation_strings":["Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0002-0199-4866","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100367774","display_name":"Xiao Yan","orcid":"https://orcid.org/0000-0002-2122-915X"},"institutions":[{"id":"https://openalex.org/I3045169105","display_name":"Southern University of Science and Technology","ror":"https://ror.org/049tv2d57","country_code":"CN","type":"education","lineage":["https://openalex.org/I3045169105"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiao Yan","raw_affiliation_strings":["Department of Computer Science and Engineering, Southern University of Science and Technology, Shenzhen, Guangdong, China"],"raw_orcid":"https://orcid.org/0000-0002-2122-915X","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Southern University of Science and Technology, Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I3045169105"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032934593","display_name":"Kaihao Ma","orcid":"https://orcid.org/0009-0001-3176-7816"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Kaihao Ma","raw_affiliation_strings":["Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113951316","display_name":"Yidi Wu","orcid":null},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yidi Wu","raw_affiliation_strings":["Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034586871","display_name":"Yuzhen Huang","orcid":"https://orcid.org/0000-0002-9536-7918"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yuzhen Huang","raw_affiliation_strings":["Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016082884","display_name":"James Cheng","orcid":"https://orcid.org/0000-0001-6313-6288"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"James Cheng","raw_affiliation_strings":["Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101812366","display_name":"Teng Su","orcid":"https://orcid.org/0009-0005-9517-2845"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Teng Su","raw_affiliation_strings":["Huawei Technologies Co. Ltd., Shenzhen, Guangdong, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Technologies Co. Ltd., Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100616402","display_name":"Fan Yu","orcid":"https://orcid.org/0000-0002-4635-5486"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fan Yu","raw_affiliation_strings":["Huawei Technologies Co. Ltd., Shenzhen, Guangdong, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Technologies Co. Ltd., Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I2250955327"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.6381,"has_fulltext":false,"cited_by_count":42,"citation_normalized_percentile":{"value":0.91447642,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":"33","issue":"8","first_page":"1967","last_page":"1981"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.917706310749054},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5577173233032227},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.49798059463500977},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.4900185167789459},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.47291049361228943},{"id":"https://openalex.org/keywords/resource-consumption","display_name":"Resource consumption","score":0.46237868070602417},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.41676467657089233},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.24843516945838928}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.917706310749054},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5577173233032227},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.49798059463500977},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.4900185167789459},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.47291049361228943},{"id":"https://openalex.org/C2777480716","wikidata":"https://www.wikidata.org/wiki/Q23582796","display_name":"Resource consumption","level":2,"score":0.46237868070602417},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.41676467657089233},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.24843516945838928},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpds.2021.3132413","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2021.3132413","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2004.10856","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2004.10856","pdf_url":"https://arxiv.org/pdf/2004.10856","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2004.10856","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2004.10856","pdf_url":"https://arxiv.org/pdf/2004.10856","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","score":0.5799999833106995,"id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":80,"referenced_works":["https://openalex.org/W1598866093","https://openalex.org/W1686810756","https://openalex.org/W1825216778","https://openalex.org/W2057332538","https://openalex.org/W2132737349","https://openalex.org/W2133564696","https://openalex.org/W2163605009","https://openalex.org/W2168231600","https://openalex.org/W2183341477","https://openalex.org/W2186615578","https://openalex.org/W2194775991","https://openalex.org/W2293634267","https://openalex.org/W2338908902","https://openalex.org/W2401231614","https://openalex.org/W2402144811","https://openalex.org/W2419597278","https://openalex.org/W2489529491","https://openalex.org/W2513383847","https://openalex.org/W2525778437","https://openalex.org/W2617411258","https://openalex.org/W2741815235","https://openalex.org/W2787998955","https://openalex.org/W2804032941","https://openalex.org/W2883830791","https://openalex.org/W2884700152","https://openalex.org/W2896457183","https://openalex.org/W2899771611","https://openalex.org/W2947664806","https://openalex.org/W2953384591","https://openalex.org/W2962776038","https://openalex.org/W2963341956","https://openalex.org/W2963351145","https://openalex.org/W2963403868","https://openalex.org/W2963959650","https://openalex.org/W2964137095","https://openalex.org/W2964174152","https://openalex.org/W2964308564","https://openalex.org/W2965658867","https://openalex.org/W2969388332","https://openalex.org/W2973727699","https://openalex.org/W2991040477","https://openalex.org/W3012514909","https://openalex.org/W3035078899","https://openalex.org/W3037639655","https://openalex.org/W3037847693","https://openalex.org/W3040573126","https://openalex.org/W3101104221","https://openalex.org/W3133969675","https://openalex.org/W3159109662","https://openalex.org/W4293404878","https://openalex.org/W4301239768","https://openalex.org/W4385245566","https://openalex.org/W6635810480","https://openalex.org/W6637373629","https://openalex.org/W6679393576","https://openalex.org/W6679434410","https://openalex.org/W6684191040","https://openalex.org/W6684859321","https://openalex.org/W6686509673","https://openalex.org/W6703652217","https://openalex.org/W6713134421","https://openalex.org/W6717177883","https://openalex.org/W6717768942","https://openalex.org/W6727690538","https://openalex.org/W6738144653","https://openalex.org/W6739901393","https://openalex.org/W6742075157","https://openalex.org/W6747917562","https://openalex.org/W6748645090","https://openalex.org/W6751349269","https://openalex.org/W6753584990","https://openalex.org/W6755207826","https://openalex.org/W6756040250","https://openalex.org/W6756379755","https://openalex.org/W6756718674","https://openalex.org/W6763650751","https://openalex.org/W6767997687","https://openalex.org/W6768723914","https://openalex.org/W6780805062","https://openalex.org/W6790981935"],"related_works":["https://openalex.org/W2045183646","https://openalex.org/W2162409446","https://openalex.org/W2109463584","https://openalex.org/W2504075107","https://openalex.org/W2354058185","https://openalex.org/W3014500535","https://openalex.org/W2319812372","https://openalex.org/W1482096554","https://openalex.org/W1595672120","https://openalex.org/W4230999561"],"abstract_inverted_index":{"Effective":[0],"parallelization":[1,23,60,127],"strategies":[2,24],"are":[3],"crucial":[4],"for":[5],"the":[6,64,73,77,90,121],"performance":[7],"of":[8,58,79],"distributed":[9,115],"deep":[10],"neural":[11],"network":[12],"(DNN)":[13],"training.":[14],"Recently,":[15],"several":[16],"methods":[17],"have":[18],"been":[19],"proposed":[20],"to":[21,62,88,112,139],"search":[22],"but":[25],"they":[26],"all":[27],"optimize":[28],"a":[29,101],"single":[30],"objective":[31],"(e.g.,":[32],"execution":[33,91],"time,":[34],"memory":[35,74],"consumption)":[36],"and":[37,83,125],"produce":[38],"only":[39],"one":[40],"strategy.":[41],"We":[42],"propose":[43],"<italic":[44,54,95,105],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[45,55,96,106],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Frontier":[46],"Tracking</i>":[47],"(FT),":[48],"an":[49],"efficient":[50],"algorithm":[51],"that":[52,132],"finds":[53],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">a":[56],"set":[57],"Pareto-optimal":[59],"strategies</i>":[61],"explore":[63],"best":[65],"trade-off":[66],"among":[67],"different":[68],"objectives.":[69],"FT":[70],"can":[71],"minimize":[72],"consumption":[75],"when":[76],"number":[78],"devices":[80],"is":[81,134],"limited":[82],"fully":[84],"utilize":[85],"additional":[86],"resources":[87],"reduce":[89],"time.":[92],"Based":[93],"on":[94],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">FT</i>":[97],",":[98,108],"we":[99],"develop":[100],"user-friendly":[102],"system,":[103],"called":[104],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">TensorOpt</i>":[107],"which":[109],"allows":[110],"users":[111],"run":[113],"their":[114],"DNN":[116],"training":[117],"jobs":[118],"without":[119],"caring":[120],"details":[122],"about":[123],"searching":[124],"coding":[126],"strategies.":[128],"Experimental":[129],"results":[130],"show":[131],"TensorOpt":[133],"more":[135],"flexible":[136],"in":[137],"adapting":[138],"resource":[140],"availability":[141],"compared":[142],"with":[143],"existing":[144],"frameworks.":[145]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":14},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":3}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2020-05-01T00:00:00"}