{"id":"https://openalex.org/W4405270867","doi":"https://doi.org/10.1109/tpds.2024.3515804","title":"UMPIPE: Unequal Microbatches-Based Pipeline Parallelism for Deep Neural Network Training","display_name":"UMPIPE: Unequal Microbatches-Based Pipeline Parallelism for Deep Neural Network Training","publication_year":2024,"publication_date":"2024-12-11","ids":{"openalex":"https://openalex.org/W4405270867","doi":"https://doi.org/10.1109/tpds.2024.3515804"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2024.3515804","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2024.3515804","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076644778","display_name":"Guangyao Zhou","orcid":"https://orcid.org/0000-0003-0809-5799"},"institutions":[{"id":"https://openalex.org/I4800084","display_name":"Southwest Jiaotong University","ror":"https://ror.org/00hn7w693","country_code":"CN","type":"education","lineage":["https://openalex.org/I4800084"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guangyao Zhou","raw_affiliation_strings":["School of Computing and Artificial Intelligence, Southwest Jiaotong University, Chengdu, Sichuan, China","School of Computer and Artificial Intelligence, Southwest Jiaotong University, China"],"raw_orcid":"https://orcid.org/0000-0003-0809-5799","affiliations":[{"raw_affiliation_string":"School of Computing and Artificial Intelligence, Southwest Jiaotong University, Chengdu, Sichuan, China","institution_ids":["https://openalex.org/I4800084"]},{"raw_affiliation_string":"School of Computer and Artificial Intelligence, Southwest Jiaotong University, China","institution_ids":["https://openalex.org/I4800084"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001104882","display_name":"Wenhong Tian","orcid":"https://orcid.org/0000-0002-5551-9796"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenhong Tian","raw_affiliation_strings":["School of Information and Software Engineering, University of Electronic Science and Technology of China, Chengdu, Sichuan, China","School of Information and Software Engineering, University of Electronic Science and Technology of China, China"],"raw_orcid":"https://orcid.org/0000-0002-5551-9796","affiliations":[{"raw_affiliation_string":"School of Information and Software Engineering, University of Electronic Science and Technology of China, Chengdu, Sichuan, China","institution_ids":["https://openalex.org/I150229711"]},{"raw_affiliation_string":"School of Information and Software Engineering, University of Electronic Science and Technology of China, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014716105","display_name":"Rajkumar Buyya","orcid":"https://orcid.org/0000-0001-9754-6496"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"The University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]},{"id":"https://openalex.org/I4210144487","display_name":"Cloud Computing Center","ror":"https://ror.org/04aa0zm65","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210144487"]}],"countries":["AU","CN"],"is_corresponding":false,"raw_author_name":"Rajkumar Buyya","raw_affiliation_strings":["Department of Computing and Information Systems, Cloud Computing and Distributed Systems (CLOUDS) Laboratory, University of Melbourne, Melbourne, VIC, Australia","Cloud Computing and Distributed Systems (CLOUDS) Laboratory, Department of Computing and Information Systems, The University of Melbourne, Australia"],"raw_orcid":"https://orcid.org/0000-0001-9754-6496","affiliations":[{"raw_affiliation_string":"Department of Computing and Information Systems, Cloud Computing and Distributed Systems (CLOUDS) Laboratory, University of Melbourne, Melbourne, VIC, Australia","institution_ids":["https://openalex.org/I4210144487"]},{"raw_affiliation_string":"Cloud Computing and Distributed Systems (CLOUDS) Laboratory, Department of Computing and Information Systems, The University of Melbourne, Australia","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100641549","display_name":"Kui Wu","orcid":"https://orcid.org/0000-0002-2069-0032"},"institutions":[{"id":"https://openalex.org/I212119943","display_name":"University of Victoria","ror":"https://ror.org/04s5mat29","country_code":"CA","type":"education","lineage":["https://openalex.org/I212119943"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Kui Wu","raw_affiliation_strings":["Department of Computer Science, University of Victoria, Victoria, BC, Canada","Department of Computer Science, the University of Victoria, Canada"],"raw_orcid":"https://orcid.org/0000-0002-2069-0032","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Victoria, Victoria, BC, Canada","institution_ids":["https://openalex.org/I212119943"]},{"raw_affiliation_string":"Department of Computer Science, the University of Victoria, Canada","institution_ids":["https://openalex.org/I212119943"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.8335,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.78706934,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":"36","issue":"2","first_page":"293","last_page":"307"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12611","display_name":"Neural Networks and Reservoir Computing","score":0.9865999817848206,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12611","display_name":"Neural Networks and Reservoir Computing","score":0.9865999817848206,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9772999882698059,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9733999967575073,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8916934728622437},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.7083215117454529},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.6805301308631897},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.6323901414871216},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5928122997283936},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5605169534683228},{"id":"https://openalex.org/keywords/data-parallelism","display_name":"Data parallelism","score":0.5215732455253601},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.3957664370536804},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.37599149346351624},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.1597452461719513}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8916934728622437},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.7083215117454529},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.6805301308631897},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6323901414871216},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5928122997283936},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5605169534683228},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.5215732455253601},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.3957664370536804},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37599149346351624},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.1597452461719513},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2024.3515804","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2024.3515804","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":39,"referenced_works":["https://openalex.org/W1479745656","https://openalex.org/W2126105956","https://openalex.org/W2883830791","https://openalex.org/W2969388332","https://openalex.org/W3016395792","https://openalex.org/W3093161730","https://openalex.org/W3102606411","https://openalex.org/W3107548300","https://openalex.org/W3130198599","https://openalex.org/W3132107458","https://openalex.org/W3151571129","https://openalex.org/W3152795099","https://openalex.org/W3153938841","https://openalex.org/W3173408461","https://openalex.org/W3175745020","https://openalex.org/W3188270315","https://openalex.org/W3195286673","https://openalex.org/W3199139386","https://openalex.org/W3202134982","https://openalex.org/W3205035978","https://openalex.org/W3211483916","https://openalex.org/W3216492891","https://openalex.org/W4206790491","https://openalex.org/W4206792915","https://openalex.org/W4210489404","https://openalex.org/W4281850905","https://openalex.org/W4285798540","https://openalex.org/W4319811780","https://openalex.org/W4321096782","https://openalex.org/W4327694855","https://openalex.org/W4327909439","https://openalex.org/W4385597806","https://openalex.org/W4401861909","https://openalex.org/W6637373629","https://openalex.org/W6756718674","https://openalex.org/W6790665546","https://openalex.org/W6799372109","https://openalex.org/W6810097121","https://openalex.org/W6811928498"],"related_works":["https://openalex.org/W2950520577","https://openalex.org/W1501159154","https://openalex.org/W1554644772","https://openalex.org/W2003935582","https://openalex.org/W2494130044","https://openalex.org/W2033862586","https://openalex.org/W3170887803","https://openalex.org/W74409296","https://openalex.org/W4230999561","https://openalex.org/W1595672120"],"abstract_inverted_index":{"The":[0],"increasing":[1],"need":[2],"for":[3,72,119,124,153,206,211,217],"large-scale":[4],"deep":[5],"neural":[6],"networks":[7],"(DNN)":[8],"has":[9],"made":[10],"parallel":[11,27,34],"training":[12,28,35,53,178,202],"an":[13],"area":[14],"of":[15,129],"intensive":[16],"focus.":[17],"One":[18],"effective":[19],"method,":[20],"microbatch-based":[21],"pipeline":[22,75],"parallelism":[23],"(notably":[24],"GPipe),":[25],"accelerates":[26,144],"in":[29,99,201],"various":[30],"architectures.":[31],"However,":[32],"existing":[33],"architectures":[36],"normally":[37],"use":[38],"equal":[39],"data":[40,80],"partitioning":[41],"(EDP),":[42],"where":[43],"each":[44],"layer's":[45],"process":[46,146],"maintains":[47],"identical":[48],"microbatch-sizes.":[49,62],"EDP":[50],"may":[51],"hinder":[52],"speed":[54],"because":[55],"different":[56],"processes":[57,84],"often":[58],"require":[59],"varying":[60],"optimal":[61],"To":[63,108],"address":[64],"this,":[65],"we":[66,113,135],"introduce":[67],"UMPIPE,":[68],"a":[69,91,138],"novel":[70],"framework":[71],"unequal":[73,79],"microbatches-based":[74],"parallelism.":[76],"UMPIPE":[77,100,120,198],"enables":[78],"partitions":[81],"(UEDP)":[82],"across":[83],"to":[85,94,195],"optimize":[86],"resource":[87],"utilization.":[88],"We":[89],"develop":[90],"recurrence":[92],"formula":[93],"calculate":[95],"the":[96,115,125,145,150,165,184,189],"time":[97,127,152],"cost":[98],"by":[101,147],"considering":[102],"both":[103],"computation":[104],"and":[105,131,156,170,188,213],"communication":[106],"processes.":[107],"further":[109],"enhance":[110],"UMPIPE's":[111],"efficiency,":[112],"propose":[114],"Dual-Chromosome":[116],"Genetic":[117],"Algorithm":[118],"(DGAP)":[121],"that":[122],"accounts":[123],"independent":[126],"costs":[128],"forward":[130],"backward":[132],"propagation.":[133],"Furthermore,":[134],"present":[136],"TiDGAP,":[137],"two-level":[139],"improvement":[140],"on":[141],"DGAP.":[142],"TiDGAP":[143,174],"simultaneously":[148],"calculating":[149],"end":[151],"multiple":[154],"individuals":[155],"microbatches":[157],"using":[158],"matrix":[159],"operations.":[160],"Our":[161],"extensive":[162],"experiments":[163],"validate":[164],"dual-chromosome":[166],"strategy's":[167],"optimization":[168],"benefits":[169],"TiDGAP's":[171],"acceleration":[172],"capabilities.":[173],"can":[175],"achieve":[176],"better":[177],"schemes":[179],"than":[180],"baselines,":[181],"such":[182],"as":[183],"local":[185],"greedy":[186],"algorithm":[187],"global":[190],"greedy-based":[191],"dynamic":[192],"programming.":[193],"Compared":[194],"(GPipe,":[196],"PipeDream),":[197],"achieves":[199],"increases":[200],"speed:":[203],"<inline-formula><tex-math":[204,208,214],"notation=\"LaTeX\">$(13.89,11.09)\\%$</tex-math></inline-formula>":[205],"GPT1-14,":[207],"notation=\"LaTeX\">$(17.11,":[209],"7.96)\\%$</tex-math></inline-formula>":[210],"VGG16":[212],"notation=\"LaTeX\">$\\geq":[215],"(170,100)\\%$</tex-math></inline-formula>":[216],"simulation":[218],"networks.":[219]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2025-12-22T23:10:17.713674","created_date":"2025-10-10T00:00:00"}