{"id":"https://openalex.org/W4382119135","doi":"https://doi.org/10.1109/tpds.2023.3281931","title":"A Survey on Auto-Parallelism of Large-Scale Deep Learning Training","display_name":"A Survey on Auto-Parallelism of Large-Scale Deep Learning Training","publication_year":2023,"publication_date":"2023-06-26","ids":{"openalex":"https://openalex.org/W4382119135","doi":"https://doi.org/10.1109/tpds.2023.3281931"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2023.3281931","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2023.3281931","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061876398","display_name":"Peng Liang","orcid":"https://orcid.org/0000-0002-5590-5179"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Peng Liang","raw_affiliation_strings":["State Key Laboratory of Parallel and Distributed Processing, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-5590-5179","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Parallel and Distributed Processing, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102970145","display_name":"Yu Tang","orcid":"https://orcid.org/0000-0002-8595-1547"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu Tang","raw_affiliation_strings":["State Key Laboratory of Parallel and Distributed Processing, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-8595-1547","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Parallel and Distributed Processing, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015232078","display_name":"Xiaoda Zhang","orcid":"https://orcid.org/0000-0003-3817-1369"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoda Zhang","raw_affiliation_strings":["Huawei Technologies Co. Ltd., Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Technologies Co. Ltd., Shenzhen, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037054290","display_name":"Youhui Bai","orcid":null},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Youhui Bai","raw_affiliation_strings":["Huawei Technologies Co. Ltd., Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Technologies Co. Ltd., Shenzhen, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101812366","display_name":"Teng Su","orcid":"https://orcid.org/0009-0005-9517-2845"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Teng Su","raw_affiliation_strings":["Huawei Technologies Co. Ltd., Shenzhen, China"],"raw_orcid":"https://orcid.org/0009-0005-9517-2845","affiliations":[{"raw_affiliation_string":"Huawei Technologies Co. Ltd., Shenzhen, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055426388","display_name":"Zhiquan Lai","orcid":"https://orcid.org/0000-0002-3458-4732"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiquan Lai","raw_affiliation_strings":["State Key Laboratory of Parallel and Distributed Processing, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-3458-4732","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Parallel and Distributed Processing, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043045076","display_name":"Linbo Qiao","orcid":"https://orcid.org/0000-0002-8285-2738"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Linbo Qiao","raw_affiliation_strings":["State Key Laboratory of Parallel and Distributed Processing, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-8285-2738","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Parallel and Distributed Processing, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100440903","display_name":"Dongsheng Li","orcid":"https://orcid.org/0000-0001-9743-2034"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongsheng Li","raw_affiliation_strings":["State Key Laboratory of Parallel and Distributed Processing, National University of Defense Technology, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0001-9743-2034","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Parallel and Distributed Processing, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5061876398"],"corresponding_institution_ids":["https://openalex.org/I170215575"],"apc_list":null,"apc_paid":null,"fwci":3.6502,"has_fulltext":false,"cited_by_count":31,"citation_normalized_percentile":{"value":0.94605988,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"34","issue":"8","first_page":"2377","last_page":"2390"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8807631731033325},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.8142803311347961},{"id":"https://openalex.org/keywords/data-parallelism","display_name":"Data parallelism","score":0.723514974117279},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5569618344306946},{"id":"https://openalex.org/keywords/task-parallelism","display_name":"Task parallelism","score":0.5358874201774597},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4828487038612366},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4489518702030182},{"id":"https://openalex.org/keywords/parallel-processing","display_name":"Parallel processing","score":0.4273768961429596},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.4259517192840576},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.40425246953964233},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.35556575655937195}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8807631731033325},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.8142803311347961},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.723514974117279},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5569618344306946},{"id":"https://openalex.org/C42992933","wikidata":"https://www.wikidata.org/wiki/Q691169","display_name":"Task parallelism","level":3,"score":0.5358874201774597},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4828487038612366},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4489518702030182},{"id":"https://openalex.org/C106515295","wikidata":"https://www.wikidata.org/wiki/Q26806595","display_name":"Parallel processing","level":2,"score":0.4273768961429596},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.4259517192840576},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.40425246953964233},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.35556575655937195},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2023.3281931","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2023.3281931","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6399999856948853,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G3606055544","display_name":null,"funder_award_id":"62025208","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":142,"referenced_works":["https://openalex.org/W1815076433","https://openalex.org/W1939156876","https://openalex.org/W1967134278","https://openalex.org/W1987188645","https://openalex.org/W2008117760","https://openalex.org/W2060393849","https://openalex.org/W2107726111","https://openalex.org/W2119717200","https://openalex.org/W2126316555","https://openalex.org/W2130416410","https://openalex.org/W2143114052","https://openalex.org/W2153386665","https://openalex.org/W2293634267","https://openalex.org/W2338908902","https://openalex.org/W2550821151","https://openalex.org/W2612076670","https://openalex.org/W2617411258","https://openalex.org/W2618530766","https://openalex.org/W2786414509","https://openalex.org/W2804682296","https://openalex.org/W2883830791","https://openalex.org/W2895891814","https://openalex.org/W2896457183","https://openalex.org/W2913888506","https://openalex.org/W2919115771","https://openalex.org/W2949869455","https://openalex.org/W2950708852","https://openalex.org/W2952915411","https://openalex.org/W2953384591","https://openalex.org/W2955454939","https://openalex.org/W2963351145","https://openalex.org/W2963358710","https://openalex.org/W2963446712","https://openalex.org/W2964137095","https://openalex.org/W2969388332","https://openalex.org/W2973727699","https://openalex.org/W2977038365","https://openalex.org/W2978633783","https://openalex.org/W2981114289","https://openalex.org/W2991040477","https://openalex.org/W3004495293","https://openalex.org/W3004659153","https://openalex.org/W3012125688","https://openalex.org/W3016430712","https://openalex.org/W3019389960","https://openalex.org/W3036703963","https://openalex.org/W3036879053","https://openalex.org/W3037699692","https://openalex.org/W3040573126","https://openalex.org/W3041621704","https://openalex.org/W3081168214","https://openalex.org/W3086105743","https://openalex.org/W3096403968","https://openalex.org/W3101104221","https://openalex.org/W3105668694","https://openalex.org/W3113588636","https://openalex.org/W3122286897","https://openalex.org/W3125012172","https://openalex.org/W3129831491","https://openalex.org/W3130395060","https://openalex.org/W3132107458","https://openalex.org/W3132977829","https://openalex.org/W3134069806","https://openalex.org/W3156636320","https://openalex.org/W3157919170","https://openalex.org/W3159649695","https://openalex.org/W3163465001","https://openalex.org/W3164253193","https://openalex.org/W3167220634","https://openalex.org/W3175745020","https://openalex.org/W3175937250","https://openalex.org/W3188270315","https://openalex.org/W3193985311","https://openalex.org/W3196884386","https://openalex.org/W3202656791","https://openalex.org/W3203426023","https://openalex.org/W3204998121","https://openalex.org/W3206314279","https://openalex.org/W3206832494","https://openalex.org/W3209444800","https://openalex.org/W3210871626","https://openalex.org/W3216598260","https://openalex.org/W4213132668","https://openalex.org/W4213255380","https://openalex.org/W4226125420","https://openalex.org/W4226479682","https://openalex.org/W4237591687","https://openalex.org/W4244566287","https://openalex.org/W4280611847","https://openalex.org/W4287750421","https://openalex.org/W4288089799","https://openalex.org/W4292779060","https://openalex.org/W4294558607","https://openalex.org/W4295312788","https://openalex.org/W4308536272","https://openalex.org/W4308536752","https://openalex.org/W4312060029","https://openalex.org/W4327694855","https://openalex.org/W4385245566","https://openalex.org/W6638545294","https://openalex.org/W6694177581","https://openalex.org/W6703652217","https://openalex.org/W6713134421","https://openalex.org/W6729508183","https://openalex.org/W6738144653","https://openalex.org/W6738964360","https://openalex.org/W6739901393","https://openalex.org/W6748687944","https://openalex.org/W6751627690","https://openalex.org/W6755207826","https://openalex.org/W6756379755","https://openalex.org/W6756718674","https://openalex.org/W6763760226","https://openalex.org/W6766978945","https://openalex.org/W6767997687","https://openalex.org/W6768142256","https://openalex.org/W6768695126","https://openalex.org/W6769627184","https://openalex.org/W6778883912","https://openalex.org/W6779863968","https://openalex.org/W6779909149","https://openalex.org/W6780239866","https://openalex.org/W6780269839","https://openalex.org/W6780805062","https://openalex.org/W6784425352","https://openalex.org/W6790665546","https://openalex.org/W6791605074","https://openalex.org/W6795223348","https://openalex.org/W6795581714","https://openalex.org/W6795742012","https://openalex.org/W6795846692","https://openalex.org/W6801859478","https://openalex.org/W6802739081","https://openalex.org/W6803433826","https://openalex.org/W6804052296","https://openalex.org/W6804279869","https://openalex.org/W6804546595","https://openalex.org/W6810596197","https://openalex.org/W6811295506","https://openalex.org/W6811928498","https://openalex.org/W6838632916","https://openalex.org/W6847431407"],"related_works":["https://openalex.org/W2950520577","https://openalex.org/W2003935582","https://openalex.org/W74409296","https://openalex.org/W1554644772","https://openalex.org/W2494130044","https://openalex.org/W29548032","https://openalex.org/W2593878938","https://openalex.org/W2468095077","https://openalex.org/W305742777","https://openalex.org/W2121547511"],"abstract_inverted_index":{"Deep":[0],"learning":[1],"(DL)":[2],"has":[3],"gained":[4],"great":[5],"success":[6,33],"in":[7,14,40,72,98,121,144,160,184,192,197],"recent":[8],"years,":[9],"leading":[10],"to":[11,47,68,93,113],"state-of-the-art":[12],"performance":[13],"research":[15],"community":[16],"and":[17,23,86,119,132,138,157,166,174,177],"industrial":[18],"fields":[19],"like":[20],"computer":[21],"vision":[22],"natural":[24],"language":[25],"processing.":[26],"One":[27],"of":[28,57,142,170],"the":[29,35,91,104],"reasons":[30],"for":[31],"this":[32,126],"is":[34,45,66,111],"huge":[36],"amount":[37],"parameters":[38,58],"adopted":[39],"DL":[41,70,96,116,145,161],"models.":[42],"However,":[43,78],"it":[44],"impractical":[46],"train":[48,69,94,114],"a":[49,54,60,130,168],"moderately":[50],"large":[51,55],"model":[52],"with":[53,74,153],"number":[56],"on":[59,135],"typical":[61],"single":[62],"device.":[63],"Thus,":[64],"It":[65],"necessary":[67],"models":[71,97,117],"clusters":[73],"distributed":[75,80],"training":[76,81],"algorithms.":[77],"traditional":[79],"algorithms":[82],"are":[83,181,195],"usually":[84],"sub-optimal":[85],"highly":[87],"customized,":[88],"which":[89,110,180,194],"owns":[90],"drawbacks":[92],"large-scale":[95,115],"varying":[99],"computing":[100,123],"clusters.":[101,124],"To":[102],"handle":[103],"above":[105],"problem,":[106],"researchers":[107],"propose":[108],"auto-parallelism,":[109],"promising":[112,196],"efficiently":[118],"practically":[120],"various":[122],"In":[125],"survey,":[127],"we":[128,148,164,188],"perform":[129],"broad":[131],"thorough":[133],"investigation":[134],"challenges,":[136],"basis,":[137],"strategy":[139],"searching":[140,178],"methods":[141,179],"auto-parallelism":[143,172,193],"training.":[146,162],"First,":[147],"abstract":[149],"basic":[150],"parallelism":[151],"schemes":[152],"their":[154],"communication":[155],"cost":[156],"memory":[158],"consumption":[159],"Further,":[163],"analyze":[165],"compare":[167],"series":[169],"current":[171],"works":[173],"investigate":[175],"strategies":[176],"commonly":[182],"used":[183],"practice.":[185],"At":[186],"last,":[187],"discuss":[189],"several":[190],"trends":[191],"further":[198],"research.":[199]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":17},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}