{"id":"https://openalex.org/W2807147113","doi":"https://doi.org/10.48550/arxiv.1806.03377","title":"PipeDream: Fast and Efficient Pipeline Parallel DNN Training","display_name":"PipeDream: Fast and Efficient Pipeline Parallel DNN Training","publication_year":2018,"publication_date":"2018-06-08","ids":{"openalex":"https://openalex.org/W2807147113","doi":"https://doi.org/10.48550/arxiv.1806.03377","mag":"2807147113"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:1806.03377","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1806.03377","pdf_url":"https://arxiv.org/pdf/1806.03377","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1806.03377","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083361305","display_name":"Aaron Harlap","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Harlap, Aaron","raw_affiliation_strings":["Microsoft Research Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Carnegie Mellon University","institution_ids":["https://openalex.org/I4210164937","https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079514101","display_name":"Deepak Narayanan","orcid":"https://orcid.org/0000-0002-3020-2848","observed_orcids":["https://orcid.org/0000-0002-3020-2848"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Narayanan, Deepak","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011102459","display_name":"Amar Phanishayee","orcid":"https://orcid.org/0009-0001-2777-1118","observed_orcids":["https://orcid.org/0009-0001-2777-1118"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Phanishayee, Amar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029415636","display_name":"Vivek Seshadri","orcid":"https://orcid.org/0009-0000-4388-4246","observed_orcids":["https://orcid.org/0009-0000-4388-4246"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Seshadri, Vivek","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060217026","display_name":"Nikhil R. Devanur","orcid":"https://orcid.org/0009-0005-4406-5935","observed_orcids":["https://orcid.org/0009-0005-4406-5935"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Devanur, Nikhil","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037557529","display_name":"Gregory R. Ganger","orcid":"https://orcid.org/0000-0002-3065-7316","observed_orcids":["https://orcid.org/0000-0002-3065-7316"]},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Ganger, Greg","raw_affiliation_strings":["Microsoft Research Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Carnegie Mellon University","institution_ids":["https://openalex.org/I4210164937","https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014824446","display_name":"Phillip B. Gibbons","orcid":"https://orcid.org/0000-0001-6967-2735","observed_orcids":["https://orcid.org/0000-0001-6967-2735"]},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Gibbons, Phil","raw_affiliation_strings":["Microsoft Research Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Carnegie Mellon University","institution_ids":["https://openalex.org/I4210164937","https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":153,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9848999977111816,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9848999977111816,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.003599999938160181,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.0017999999690800905,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pipeline-parallelism","display_name":"pipeline parallelism","score":0.9649999737739563},{"id":"https://openalex.org/keywords/deep-neural-network-training","display_name":"deep neural network training","score":0.878000020980835},{"id":"https://openalex.org/keywords/gpu-clusters","display_name":"GPU clusters","score":0.6510000228881836},{"id":"https://openalex.org/keywords/communication-to-computation-ratio","display_name":"communication-to-computation ratio","score":0.7289999723434448},{"id":"https://openalex.org/keywords/data-parallel-training","display_name":"data-parallel training","score":0.7229999899864197},{"id":"https://openalex.org/keywords/layer-partitioning","display_name":"layer partitioning","score":0.7509999871253967},{"id":"https://openalex.org/keywords/communication-overlap","display_name":"communication overlap","score":0.41600000858306885},{"id":"https://openalex.org/keywords/distributed-training","display_name":"distributed training","score":0.4410000145435333}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8684484958648682},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.7821984887123108},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.7646349668502808},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.7017134428024292},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6764397621154785},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.5548515915870667},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5474541187286377},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.5210758447647095},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.4308403730392456},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.3660743832588196},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.34086930751800537},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.27309930324554443},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.23380938172340393},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.09080782532691956},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:1806.03377","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1806.03377","pdf_url":"https://arxiv.org/pdf/1806.03377","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2807147113","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/1806.03377","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1806.03377","doi":"https://doi.org/10.48550/arxiv.1806.03377","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1806.03377","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1806.03377","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1806.03377","pdf_url":"https://arxiv.org/pdf/1806.03377","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":4,"referenced_works":["https://openalex.org/W1686810756","https://openalex.org/W2168231600","https://openalex.org/W2949300694","https://openalex.org/W2950094539"],"related_works":["https://openalex.org/W2969388332","https://openalex.org/W4400064699","https://openalex.org/W7057068629","https://openalex.org/W3116045160","https://openalex.org/W2891017939","https://openalex.org/W4224441437","https://openalex.org/W4410087408","https://openalex.org/W3192514716","https://openalex.org/W4385597806","https://openalex.org/W3031276512"],"abstract_inverted_index":{"PipeDream":[0,42,64,120],"is":[1,121],"a":[2],"Deep":[3],"Neural":[4],"Network(DNN)":[5],"training":[6,30],"system":[7],"for":[8,49,86],"GPUs":[9,68],"that":[10,119],"parallelizes":[11],"computation":[12],"by":[13,28,45,70],"pipelining":[14],"execution":[15],"across":[16],"multiple":[17],"machines.":[18],"Its":[19],"pipeline":[20],"parallel":[21],"computing":[22],"model":[23,84],"avoids":[24],"the":[25,92],"slowdowns":[26],"faced":[27],"data-parallel":[29,54,130],"when":[31],"large":[32,50],"models":[33],"and/or":[34],"limited":[35],"network":[36],"bandwidth":[37],"induce":[38],"high":[39],"communication-to-computation":[40],"ratios.":[41],"reduces":[43],"communication":[44,61],"up":[46,122],"to":[47,53,77,103,106,123,129],"95%":[48],"DNNs":[51,113],"relative":[52],"training,":[55],"and":[56,62,80,90,94],"allows":[57],"perfect":[58],"overlap":[59],"of":[60,97],"computation.":[63],"keeps":[65],"all":[66],"available":[67],"productive":[69],"systematically":[71],"partitioning":[72],"DNN":[73],"layers":[74],"among":[75],"them":[76],"balance":[78],"work":[79],"minimize":[81],"communication,":[82],"versions":[83],"parameters":[85],"backward":[87,95],"pass":[88],"correctness,":[89],"schedules":[91],"forward":[93],"passes":[96],"different":[98,112,116],"inputs":[99],"in":[100,126],"round-robin":[101],"fashion":[102],"optimize":[104],"\"time":[105],"target":[107],"accuracy\".":[108],"Experiments":[109],"with":[110],"five":[111],"on":[114],"two":[115],"clusters":[117],"show":[118],"5x":[124],"faster":[125],"time-to-accuracy":[127],"compared":[128],"training.":[131]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":20},{"year":2023,"cited_by_count":24},{"year":2022,"cited_by_count":16},{"year":2021,"cited_by_count":31},{"year":2020,"cited_by_count":27},{"year":2019,"cited_by_count":24},{"year":2018,"cited_by_count":4}],"updated_date":"2026-10-31T12:09:11.096373","created_date":"2018-06-13T00:00:00"}