{"id":"https://openalex.org/W4292951128","doi":"https://doi.org/10.1142/s0129626422500050","title":"BaPipe: Balanced Pipeline Parallelism for DNN Training","display_name":"BaPipe: Balanced Pipeline Parallelism for DNN Training","publication_year":2022,"publication_date":"2022-08-19","ids":{"openalex":"https://openalex.org/W4292951128","doi":"https://doi.org/10.1142/s0129626422500050"},"language":"en","primary_location":{"id":"doi:10.1142/s0129626422500050","is_oa":false,"landing_page_url":"https://doi.org/10.1142/s0129626422500050","pdf_url":null,"source":{"id":"https://openalex.org/S18360026","display_name":"Parallel Processing Letters","issn_l":"0129-6264","issn":["0129-6264","1793-642X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319815","host_organization_name":"World Scientific","host_organization_lineage":["https://openalex.org/P4310319815"],"host_organization_lineage_names":["World Scientific"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Parallel Processing Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102507634","display_name":"Letian Zhao","orcid":null},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210119392","display_name":"Institute of Microelectronics","ror":"https://ror.org/02s6gs133","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210119392"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Letian Zhao","raw_affiliation_strings":["Institute of Microelectronics, Department of Physics, University of Science and Technology of China, Hefei 230026, China","State Key Laboratory of Particle Detection and Electronics, University of Science and Technology of China, Hefei 230026, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Microelectronics, Department of Physics, University of Science and Technology of China, Hefei 230026, China","institution_ids":["https://openalex.org/I126520041","https://openalex.org/I4210119392"]},{"raw_affiliation_string":"State Key Laboratory of Particle Detection and Electronics, University of Science and Technology of China, Hefei 230026, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101888899","display_name":"Rui Xu","orcid":"https://orcid.org/0000-0002-5549-236X"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210119392","display_name":"Institute of Microelectronics","ror":"https://ror.org/02s6gs133","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210119392"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rui Xu","raw_affiliation_strings":["Institute of Microelectronics, Department of Physics, University of Science and Technology of China, Hefei 230026, China","State Key Laboratory of Particle Detection and Electronics, University of Science and Technology of China, Hefei 230026, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Microelectronics, Department of Physics, University of Science and Technology of China, Hefei 230026, China","institution_ids":["https://openalex.org/I126520041","https://openalex.org/I4210119392"]},{"raw_affiliation_string":"State Key Laboratory of Particle Detection and Electronics, University of Science and Technology of China, Hefei 230026, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100734913","display_name":"Tianqi Wang","orcid":"https://orcid.org/0000-0002-5921-6565"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210119392","display_name":"Institute of Microelectronics","ror":"https://ror.org/02s6gs133","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210119392"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianqi Wang","raw_affiliation_strings":["Institute of Microelectronics, Department of Physics, University of Science and Technology of China, Hefei 230026, China","State Key Laboratory of Particle Detection and Electronics, University of Science and Technology of China, Hefei 230026, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Microelectronics, Department of Physics, University of Science and Technology of China, Hefei 230026, China","institution_ids":["https://openalex.org/I126520041","https://openalex.org/I4210119392"]},{"raw_affiliation_string":"State Key Laboratory of Particle Detection and Electronics, University of Science and Technology of China, Hefei 230026, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017010290","display_name":"Teng Tian","orcid":"https://orcid.org/0000-0002-0594-5957"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210119392","display_name":"Institute of Microelectronics","ror":"https://ror.org/02s6gs133","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210119392"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Teng Tian","raw_affiliation_strings":["Institute of Microelectronics, Department of Physics, University of Science and Technology of China, Hefei 230026, China","State Key Laboratory of Particle Detection and Electronics, University of Science and Technology of China, Hefei 230026, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Microelectronics, Department of Physics, University of Science and Technology of China, Hefei 230026, China","institution_ids":["https://openalex.org/I126520041","https://openalex.org/I4210119392"]},{"raw_affiliation_string":"State Key Laboratory of Particle Detection and Electronics, University of Science and Technology of China, Hefei 230026, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100395727","display_name":"Xiaotian Wang","orcid":"https://orcid.org/0000-0002-6970-2997"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210119392","display_name":"Institute of Microelectronics","ror":"https://ror.org/02s6gs133","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210119392"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaotian Wang","raw_affiliation_strings":["Institute of Microelectronics, Department of Physics, University of Science and Technology of China, Hefei 230026, China","State Key Laboratory of Particle Detection and Electronics, University of Science and Technology of China, Hefei 230026, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Microelectronics, Department of Physics, University of Science and Technology of China, Hefei 230026, China","institution_ids":["https://openalex.org/I126520041","https://openalex.org/I4210119392"]},{"raw_affiliation_string":"State Key Laboratory of Particle Detection and Electronics, University of Science and Technology of China, Hefei 230026, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027254893","display_name":"Wei Wu","orcid":"https://orcid.org/0000-0002-2750-6365"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210119392","display_name":"Institute of Microelectronics","ror":"https://ror.org/02s6gs133","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210119392"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Wu","raw_affiliation_strings":["Institute of Microelectronics, Department of Physics, University of Science and Technology of China, Hefei 230026, China","State Key Laboratory of Particle Detection and Electronics, University of Science and Technology of China, Hefei 230026, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Microelectronics, Department of Physics, University of Science and Technology of China, Hefei 230026, China","institution_ids":["https://openalex.org/I126520041","https://openalex.org/I4210119392"]},{"raw_affiliation_string":"State Key Laboratory of Particle Detection and Electronics, University of Science and Technology of China, Hefei 230026, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062473941","display_name":"Chio-in Ieong","orcid":null},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chio-In Ieong","raw_affiliation_strings":["Huawei Technologies, Shenzhen 518129, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Technologies, Shenzhen 518129, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079471306","display_name":"Xi Jin","orcid":"https://orcid.org/0000-0002-4159-2925"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210119392","display_name":"Institute of Microelectronics","ror":"https://ror.org/02s6gs133","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210119392"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xi Jin","raw_affiliation_strings":["Institute of Microelectronics, Department of Physics, University of Science and Technology of China, Hefei 230026, China","State Key Laboratory of Particle Detection and Electronics, University of Science and Technology of China, Hefei 230026, China"],"raw_orcid":"https://orcid.org/0000-0002-4159-2925","affiliations":[{"raw_affiliation_string":"Institute of Microelectronics, Department of Physics, University of Science and Technology of China, Hefei 230026, China","institution_ids":["https://openalex.org/I126520041","https://openalex.org/I4210119392"]},{"raw_affiliation_string":"State Key Laboratory of Particle Detection and Electronics, University of Science and Technology of China, Hefei 230026, China","institution_ids":["https://openalex.org/I126520041"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5079471306"],"corresponding_institution_ids":["https://openalex.org/I126520041","https://openalex.org/I4210119392"],"apc_list":null,"apc_paid":null,"fwci":0.5669,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.62755102,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"32","issue":"03n04","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8758664131164551},{"id":"https://openalex.org/keywords/data-parallelism","display_name":"Data parallelism","score":0.8004183769226074},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.7323427200317383},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6721969842910767},{"id":"https://openalex.org/keywords/partition","display_name":"Partition (number theory)","score":0.6489381790161133},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.6064242124557495},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.5279273986816406},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.48437318205833435},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.44888415932655334},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.324370414018631},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.24674654006958008},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.22327566146850586},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.06676846742630005}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8758664131164551},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.8004183769226074},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.7323427200317383},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6721969842910767},{"id":"https://openalex.org/C42812","wikidata":"https://www.wikidata.org/wiki/Q1082910","display_name":"Partition (number theory)","level":2,"score":0.6489381790161133},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.6064242124557495},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.5279273986816406},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.48437318205833435},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.44888415932655334},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.324370414018631},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.24674654006958008},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.22327566146850586},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.06676846742630005},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1142/s0129626422500050","is_oa":false,"landing_page_url":"https://doi.org/10.1142/s0129626422500050","pdf_url":null,"source":{"id":"https://openalex.org/S18360026","display_name":"Parallel Processing Letters","issn_l":"0129-6264","issn":["0129-6264","1793-642X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319815","host_organization_name":"World Scientific","host_organization_lineage":["https://openalex.org/P4310319815"],"host_organization_lineage_names":["World Scientific"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Parallel Processing Letters","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6741947268","display_name":null,"funder_award_id":"HO2018085418","funder_id":"https://openalex.org/F4320322183","funder_display_name":"Huawei Technologies"}],"funders":[{"id":"https://openalex.org/F4320322183","display_name":"Huawei Technologies","ror":"https://ror.org/00cmhce21"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W2124446952","https://openalex.org/W2155893237","https://openalex.org/W2194775991","https://openalex.org/W2302255633","https://openalex.org/W2612690371","https://openalex.org/W2618530766","https://openalex.org/W2626991402","https://openalex.org/W2734941459","https://openalex.org/W2807778630","https://openalex.org/W2890068895","https://openalex.org/W2904445802","https://openalex.org/W2950592884","https://openalex.org/W2969388332","https://openalex.org/W2969766737","https://openalex.org/W2972087877","https://openalex.org/W2973058717","https://openalex.org/W2982408761","https://openalex.org/W3004495293","https://openalex.org/W4212774754"],"related_works":["https://openalex.org/W2950520577","https://openalex.org/W1501159154","https://openalex.org/W1554644772","https://openalex.org/W2003935582","https://openalex.org/W2494130044","https://openalex.org/W2033862586","https://openalex.org/W3170887803","https://openalex.org/W2963831937","https://openalex.org/W74409296","https://openalex.org/W3209384898"],"abstract_inverted_index":{"The":[0],"size":[1],"of":[2,12,33,90,103,115,125,148,177,196,204],"deep":[3,19],"neural":[4],"networks":[5,91],"(DNNs)":[6],"grows":[7],"rapidly":[8],"as":[9,106,108,160],"the":[10,13,31,81,87,101,109,121,146,174,182],"complexity":[11],"machine":[14],"learning":[15,20],"algorithm":[16],"increases.":[17],"Distributed":[18],"based":[21],"on":[22,72,169,207],"model":[23],"parallelism":[24,51,64,97,187],"has":[25],"been":[26],"widely":[27],"used":[28],"to":[29,37,61,92,140],"satisfy":[30],"requirements":[32],"DNN":[34,70,104],"training":[35,47,71],"related":[36],"computation":[38],"and":[39,65,83,112,153,163,172,189,200,210],"memory.":[40],"In":[41,76],"this":[42],"paper,":[43],"we":[44],"propose":[45],"a":[46,137,194],"framework":[48],"for":[49,69,86,185],"pipeline":[50,63,96,126,190],"called":[52],"BaPipe":[53,118,192],"(Balanced":[54],"Pipeline)":[55],"that":[56],"can":[57],"automatically":[58,119,141],"explore":[59],"methods":[60],"schedule":[62],"balanced":[66],"partition":[67,89],"strategies":[68],"heterogeneous":[73,211],"accelerator":[74,79],"clusters.":[75,179],"BaPipe,":[77],"each":[78,116],"calculates":[80],"forward":[82],"backward":[84],"propagation":[85],"assigned":[88],"implement":[93],"an":[94],"intra-batch":[95],"strategy.":[98],"By":[99],"considering":[100],"parameters":[102],"models":[105],"well":[107],"computation,":[110],"memory,":[111],"communication":[113],"resources":[114],"accelerator,":[117],"selects":[120],"most":[122],"suitable":[123],"method":[124],"scheduling":[127,132],"from":[128],"among":[129],"multiple":[130],"proposed":[131],"modes.":[133],"It":[134],"also":[135],"uses":[136],"novel":[138],"strategy":[139],"investigate":[142],"load":[143],"balancing":[144],"in":[145],"context":[147],"inter-layer":[149],"partition,":[150,152],"intra-layer":[151],"coarse-grained":[154],"partition.":[155],"We":[156],"trained":[157],"such":[158],"DNNs":[159],"VGG-16,":[161],"ResNet-50,":[162],"Google\u2019s":[164],"Neural":[165],"Machine":[166],"Translation":[167],"(GNMT)":[168],"GPU":[170],"clusters,":[171],"simulated":[173],"training-related":[175],"performance":[176],"FPGA":[178],"Compared":[180],"with":[181],"state-of-the-art":[183],"frameworks":[184],"data":[186],"(DP)":[188],"parallelism,":[191],"provides":[193],"speedup":[195],"[Formula:":[197,201],"see":[198,202],"text]":[199,203],"memory":[205],"reduction":[206],"various":[208],"homogeneous":[209],"platforms.":[212]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}