{"id":"https://openalex.org/W3086105743","doi":"https://doi.org/10.14778/3415478.3415530","title":"PyTorch distributed","display_name":"PyTorch distributed","publication_year":2020,"publication_date":"2020-08-01","ids":{"openalex":"https://openalex.org/W3086105743","doi":"https://doi.org/10.14778/3415478.3415530","mag":"3086105743"},"language":"en","primary_location":{"id":"doi:10.14778/3415478.3415530","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3415478.3415530","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042802004","display_name":"Li Shen","orcid":"https://orcid.org/0000-0001-5659-3464"},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Shen Li","raw_affiliation_strings":["Facebook AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101480914","display_name":"Yanli Zhao","orcid":"https://orcid.org/0000-0002-9821-9830"},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Yanli Zhao","raw_affiliation_strings":["Facebook AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014393716","display_name":"Rohan Varma","orcid":"https://orcid.org/0000-0002-4635-7334"},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Rohan Varma","raw_affiliation_strings":["Facebook AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089657833","display_name":"Omkar Salpekar","orcid":null},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Omkar Salpekar","raw_affiliation_strings":["Facebook AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010924039","display_name":"Pieter Noordhuis","orcid":null},"institutions":[{"id":"https://openalex.org/I4654613","display_name":"University of Warsaw","ror":"https://ror.org/039bjqg32","country_code":"PL","type":"education","lineage":["https://openalex.org/I4654613"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Pieter Noordhuis","raw_affiliation_strings":["University of Warsaw"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Warsaw","institution_ids":["https://openalex.org/I4654613"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100416750","display_name":"Teng Li","orcid":"https://orcid.org/0000-0003-0111-0108"},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Teng Li","raw_affiliation_strings":["Facebook AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004759214","display_name":"Adam Paszke","orcid":"https://orcid.org/0000-0002-7665-4559"},"institutions":[{"id":"https://openalex.org/I4654613","display_name":"University of Warsaw","ror":"https://ror.org/039bjqg32","country_code":"PL","type":"education","lineage":["https://openalex.org/I4654613"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Adam Paszke","raw_affiliation_strings":["University of Warsaw"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Warsaw","institution_ids":["https://openalex.org/I4654613"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055268532","display_name":"Jeff Smith","orcid":"https://orcid.org/0000-0001-8445-586X"},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Jeff Smith","raw_affiliation_strings":["Facebook AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042323616","display_name":"Brian Vaughan","orcid":"https://orcid.org/0000-0002-0201-7442"},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Brian Vaughan","raw_affiliation_strings":["Facebook AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003360849","display_name":"Pritam Damania","orcid":null},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Pritam Damania","raw_affiliation_strings":["Facebook AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5002078510","display_name":"Soumith Chintala","orcid":"https://orcid.org/0000-0003-2147-9850"},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Soumith Chintala","raw_affiliation_strings":["Facebook AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI","institution_ids":["https://openalex.org/I2252078561"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":11,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":21.9365,"has_fulltext":false,"cited_by_count":423,"citation_normalized_percentile":{"value":0.99576814,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"13","issue":"12","first_page":"3005","last_page":"3018"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8740688562393188},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7619080543518066},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.5684854984283447},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.5569928884506226},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.5493528246879578},{"id":"https://openalex.org/keywords/data-parallelism","display_name":"Data parallelism","score":0.5272340178489685},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.5164749622344971},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.49228182435035706},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.46981576085090637},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4127231538295746},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.13709649443626404}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8740688562393188},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7619080543518066},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5684854984283447},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.5569928884506226},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.5493528246879578},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.5272340178489685},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5164749622344971},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.49228182435035706},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.46981576085090637},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4127231538295746},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.13709649443626404},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.14778/3415478.3415530","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3415478.3415530","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.4099999964237213,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W569156507","https://openalex.org/W1597009674","https://openalex.org/W2083842231","https://openalex.org/W2137028279","https://openalex.org/W2194775991","https://openalex.org/W2407022425","https://openalex.org/W2766086266","https://openalex.org/W2767094836","https://openalex.org/W2906007643","https://openalex.org/W2944793600","https://openalex.org/W2969388332","https://openalex.org/W2970971581","https://openalex.org/W2975712713","https://openalex.org/W2981114289","https://openalex.org/W2994144272","https://openalex.org/W3047537431","https://openalex.org/W3129831491","https://openalex.org/W4212774754"],"related_works":["https://openalex.org/W2950520577","https://openalex.org/W1554644772","https://openalex.org/W2494130044","https://openalex.org/W2003935582","https://openalex.org/W3209384898","https://openalex.org/W74409296","https://openalex.org/W1991844655","https://openalex.org/W1595834484","https://openalex.org/W3170887803","https://openalex.org/W2468095077"],"abstract_inverted_index":{"This":[0],"paper":[1],"presents":[2],"the":[3,9,36,46,78,85,109,113,115,127,161],"design,":[4],"implementation,":[5],"and":[6,27,41,73,95,120,151],"evaluation":[7],"of":[8,38,80,112,132],"PyTorch":[10,134,162],"distributed":[11,66,81,128,141,163],"data":[12,82,142,164],"parallel":[13,165],"module.":[14],"Py-Torch":[15],"is":[16],"a":[17,62],"widely-adopted":[18],"scientific":[19],"computing":[20],"package":[21],"used":[22],"in":[23,31],"deep":[24,32],"learning":[25,33],"research":[26],"applications.":[28],"Recent":[29],"advances":[30],"argue":[34],"for":[35,65],"value":[37],"large":[39,42],"datasets":[40],"models,":[43],"which":[44],"necessitates":[45],"ability":[47],"to":[48,53,69,91,103,125,139],"scale":[49],"out":[50],"model":[51,86,105],"training":[52,67,129],"more":[54],"computational":[55,89],"resources.":[56],"Data":[57],"parallelism":[58,83],"has":[59],"emerged":[60],"as":[61],"popular":[63],"solution":[64],"thanks":[68],"its":[70],"straightforward":[71],"principle":[72],"broad":[74],"applicability.":[75],"In":[76],"general,":[77],"technique":[79],"replicates":[84],"on":[87],"every":[88],"resource":[90],"generate":[92],"gradients":[93,99],"independently":[94],"then":[96],"communicates":[97],"those":[98],"at":[100],"each":[101],"iteration":[102],"keep":[104],"replicas":[106],"consistent.":[107],"Despite":[108],"conceptual":[110],"simplicity":[111],"technique,":[114],"subtle":[116],"dependencies":[117],"between":[118],"computation":[119,148],"communication":[121],"make":[122],"it":[123],"non-trivial":[124],"optimize":[126],"efficiency.":[130],"As":[131],"v1.5,":[133],"natively":[135],"provides":[136],"several":[137],"techniques":[138],"accelerate":[140],"parallel,":[143],"including":[144],"bucketing":[145],"gradients,":[146],"overlapping":[147],"with":[149],"communication,":[150],"skipping":[152],"gradient":[153],"synchronization.":[154],"Evaluations":[155],"show":[156],"that,":[157],"when":[158],"configured":[159],"appropriately,":[160],"module":[166],"attains":[167],"near-linear":[168],"scalability":[169],"using":[170],"256":[171],"GPUs.":[172]},"counts_by_year":[{"year":2026,"cited_by_count":22},{"year":2025,"cited_by_count":139},{"year":2024,"cited_by_count":100},{"year":2023,"cited_by_count":80},{"year":2022,"cited_by_count":51},{"year":2021,"cited_by_count":29},{"year":2020,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}