{"id":"https://openalex.org/W2972087877","doi":"https://doi.org/10.1145/3320060","title":"Demystifying Parallel and Distributed Deep Learning","display_name":"Demystifying Parallel and Distributed Deep Learning","publication_year":2019,"publication_date":"2019-08-30","ids":{"openalex":"https://openalex.org/W2972087877","doi":"https://doi.org/10.1145/3320060","mag":"2972087877"},"language":"en","primary_location":{"id":"doi:10.1145/3320060","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3320060","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"type":"review","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002615744","display_name":"Tal Ben\u2010Nun","orcid":"https://orcid.org/0000-0002-3657-6568"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":true,"raw_author_name":"Tal Ben-Nun","raw_affiliation_strings":["ETH Zurich, Z\u00fcrich, Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Zurich, Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5026990786","display_name":"Torsten Hoefler","orcid":"https://orcid.org/0000-0002-1333-9797"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Torsten Hoefler","raw_affiliation_strings":["ETH Zurich, Z\u00fcrich, Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Zurich, Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5002615744"],"corresponding_institution_ids":["https://openalex.org/I35440088"],"apc_list":null,"apc_paid":null,"fwci":47.5427,"has_fulltext":false,"cited_by_count":559,"citation_normalized_percentile":{"value":0.99858998,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"52","issue":"4","first_page":"1","last_page":"43"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.9022949934005737},{"id":"https://openalex.org/keywords/asynchronous-communication","display_name":"Asynchronous communication","score":0.767254114151001},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.6741570830345154},{"id":"https://openalex.org/keywords/concurrency","display_name":"Concurrency","score":0.6252843737602234},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5538321733474731},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.5425238609313965},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5390481352806091},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.5264903903007507},{"id":"https://openalex.org/keywords/data-parallelism","display_name":"Data parallelism","score":0.5133320689201355},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.4647497534751892},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.44948774576187134},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.384888231754303},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.37902528047561646},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.37046515941619873},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.35498470067977905}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.9022949934005737},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.767254114151001},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.6741570830345154},{"id":"https://openalex.org/C193702766","wikidata":"https://www.wikidata.org/wiki/Q1414548","display_name":"Concurrency","level":2,"score":0.6252843737602234},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5538321733474731},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.5425238609313965},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5390481352806091},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.5264903903007507},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.5133320689201355},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.4647497534751892},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.44948774576187134},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.384888231754303},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37902528047561646},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.37046515941619873},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.35498470067977905},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3320060","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3320060","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1374109794","display_name":null,"funder_award_id":"678880","funder_id":"https://openalex.org/F4320338335","funder_display_name":"H2020 European Research Council"},{"id":"https://openalex.org/G2749364179","display_name":null,"funder_award_id":"COFUND","funder_id":"https://openalex.org/F4320338368","funder_display_name":"FP7 People: Marie-Curie Actions"}],"funders":[{"id":"https://openalex.org/F4320338335","display_name":"H2020 European Research Council","ror":"https://ror.org/0472cxd90"},{"id":"https://openalex.org/F4320338368","display_name":"FP7 People: Marie-Curie Actions","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":302,"referenced_works":["https://openalex.org/W22040386","https://openalex.org/W196761320","https://openalex.org/W326710513","https://openalex.org/W587794757","https://openalex.org/W607505555","https://openalex.org/W619799124","https://openalex.org/W753012316","https://openalex.org/W778657980","https://openalex.org/W809736386","https://openalex.org/W988471318","https://openalex.org/W1005811612","https://openalex.org/W1442374986","https://openalex.org/W1487564550","https://openalex.org/W1493774699","https://openalex.org/W1514535095","https://openalex.org/W1530262073","https://openalex.org/W1533861849","https://openalex.org/W1534477342","https://openalex.org/W1552387502","https://openalex.org/W1568834902","https://openalex.org/W1570056235","https://openalex.org/W1572016165","https://openalex.org/W1583837637","https://openalex.org/W1588544414","https://openalex.org/W1598866093","https://openalex.org/W1604973310","https://openalex.org/W1613249581","https://openalex.org/W1667652561","https://openalex.org/W1677182931","https://openalex.org/W1762484328","https://openalex.org/W1821462560","https://openalex.org/W1841592590","https://openalex.org/W1867117070","https://openalex.org/W1902934009","https://openalex.org/W1966287322","https://openalex.org/W1976981886","https://openalex.org/W1980287119","https://openalex.org/W1982633241","https://openalex.org/W1988307172","https://openalex.org/W1992208280","https://openalex.org/W1992432622","https://openalex.org/W1994616650","https://openalex.org/W2016559894","https://openalex.org/W2018162709","https://openalex.org/W2035476608","https://openalex.org/W2039548408","https://openalex.org/W2053637704","https://openalex.org/W2057332538","https://openalex.org/W2060108852","https://openalex.org/W2060393849","https://openalex.org/W2064675550","https://openalex.org/W2083842231","https://openalex.org/W2086161653","https://openalex.org/W2087402357","https://openalex.org/W2090690379","https://openalex.org/W2094233035","https://openalex.org/W2097117768","https://openalex.org/W2097215900","https://openalex.org/W2098147619","https://openalex.org/W2101517602","https://openalex.org/W2102017903","https://openalex.org/W2102061396","https://openalex.org/W2107438106","https://openalex.org/W2107582394","https://openalex.org/W2107789863","https://openalex.org/W2107878631","https://openalex.org/W2108598243","https://openalex.org/W2109722477","https://openalex.org/W2110485445","https://openalex.org/W2110798204","https://openalex.org/W2112796928","https://openalex.org/W2112972442","https://openalex.org/W2117499659","https://openalex.org/W2119717200","https://openalex.org/W2120432001","https://openalex.org/W2126715624","https://openalex.org/W2130062883","https://openalex.org/W2130566259","https://openalex.org/W2132737349","https://openalex.org/W2134797427","https://openalex.org/W2136922672","https://openalex.org/W2141473882","https://openalex.org/W2142354294","https://openalex.org/W2143649445","https://openalex.org/W2146502635","https://openalex.org/W2147800946","https://openalex.org/W2147860648","https://openalex.org/W2149600041","https://openalex.org/W2150355110","https://openalex.org/W2152839228","https://openalex.org/W2154834860","https://openalex.org/W2155893237","https://openalex.org/W2157331557","https://openalex.org/W2161846260","https://openalex.org/W2162390675","https://openalex.org/W2163605009","https://openalex.org/W2164278908","https://openalex.org/W2165698076","https://openalex.org/W2165808215","https://openalex.org/W2166706236","https://openalex.org/W2168231600","https://openalex.org/W2172654076","https://openalex.org/W2172734211","https://openalex.org/W2173213060","https://openalex.org/W2177838837","https://openalex.org/W2182445740","https://openalex.org/W2193413348","https://openalex.org/W2194775991","https://openalex.org/W2204075824","https://openalex.org/W2249098512","https://openalex.org/W2253807446","https://openalex.org/W2260663238","https://openalex.org/W2271840356","https://openalex.org/W2278433511","https://openalex.org/W2279098554","https://openalex.org/W2284050935","https://openalex.org/W2284143396","https://openalex.org/W2286848013","https://openalex.org/W2294929133","https://openalex.org/W2300242332","https://openalex.org/W2304648132","https://openalex.org/W2319920447","https://openalex.org/W2327501763","https://openalex.org/W2338908902","https://openalex.org/W2339765813","https://openalex.org/W2342173569","https://openalex.org/W2342840547","https://openalex.org/W2405578611","https://openalex.org/W2405883473","https://openalex.org/W2405920868","https://openalex.org/W2407022425","https://openalex.org/W2412412865","https://openalex.org/W2423581336","https://openalex.org/W2436219157","https://openalex.org/W2436522418","https://openalex.org/W2469490737","https://openalex.org/W2474388053","https://openalex.org/W2510697685","https://openalex.org/W2518108298","https://openalex.org/W2530887700","https://openalex.org/W2553303224","https://openalex.org/W2553581924","https://openalex.org/W2554096094","https://openalex.org/W2555706138","https://openalex.org/W2557113115","https://openalex.org/W2557191228","https://openalex.org/W2560674852","https://openalex.org/W2561036008","https://openalex.org/W2561801540","https://openalex.org/W2562677074","https://openalex.org/W2563343794","https://openalex.org/W2565851976","https://openalex.org/W2572396979","https://openalex.org/W2573285109","https://openalex.org/W2578027111","https://openalex.org/W2579247884","https://openalex.org/W2580688187","https://openalex.org/W2581399127","https://openalex.org/W2583383421","https://openalex.org/W2585029434","https://openalex.org/W2592417500","https://openalex.org/W2593744649","https://openalex.org/W2604292070","https://openalex.org/W2604319603","https://openalex.org/W2604738573","https://openalex.org/W2604783387","https://openalex.org/W2605739168","https://openalex.org/W2606722458","https://openalex.org/W2606891064","https://openalex.org/W2607662938","https://openalex.org/W2608239929","https://openalex.org/W2610817424","https://openalex.org/W2612026221","https://openalex.org/W2612445135","https://openalex.org/W2613168994","https://openalex.org/W2617242334","https://openalex.org/W2617766261","https://openalex.org/W2618818856","https://openalex.org/W2620481664","https://openalex.org/W2622263826","https://openalex.org/W2625885504","https://openalex.org/W2626721309","https://openalex.org/W2626792426","https://openalex.org/W2648403730","https://openalex.org/W2725513608","https://openalex.org/W2733710724","https://openalex.org/W2735075612","https://openalex.org/W2735913953","https://openalex.org/W2745269232","https://openalex.org/W2746842762","https://openalex.org/W2747469359","https://openalex.org/W2747482047","https://openalex.org/W2747590145","https://openalex.org/W2748513770","https://openalex.org/W2750933313","https://openalex.org/W2751836095","https://openalex.org/W2752512710","https://openalex.org/W2752689052","https://openalex.org/W2757910899","https://openalex.org/W2759465730","https://openalex.org/W2765602917","https://openalex.org/W2766164908","https://openalex.org/W2766447205","https://openalex.org/W2766527293","https://openalex.org/W2767077319","https://openalex.org/W2769644379","https://openalex.org/W2769653148","https://openalex.org/W2770298516","https://openalex.org/W2771727678","https://openalex.org/W2775776326","https://openalex.org/W2778749116","https://openalex.org/W2782417188","https://openalex.org/W2785366763","https://openalex.org/W2785430118","https://openalex.org/W2785452945","https://openalex.org/W2785856116","https://openalex.org/W2786320458","https://openalex.org/W2787513823","https://openalex.org/W2788183263","https://openalex.org/W2788853733","https://openalex.org/W2790598069","https://openalex.org/W2810075754","https://openalex.org/W2884711234","https://openalex.org/W2885142640","https://openalex.org/W2899366056","https://openalex.org/W2900167092","https://openalex.org/W2900824371","https://openalex.org/W2901299405","https://openalex.org/W2901541570","https://openalex.org/W2903482050","https://openalex.org/W2912500072","https://openalex.org/W2913954081","https://openalex.org/W2914238450","https://openalex.org/W2914584698","https://openalex.org/W2919115771","https://openalex.org/W2936995161","https://openalex.org/W2949117887","https://openalex.org/W2949264490","https://openalex.org/W2949966795","https://openalex.org/W2950178297","https://openalex.org/W2950182411","https://openalex.org/W2950894517","https://openalex.org/W2951583185","https://openalex.org/W2951781666","https://openalex.org/W2951810231","https://openalex.org/W2951886768","https://openalex.org/W2952033860","https://openalex.org/W2952388062","https://openalex.org/W2952716993","https://openalex.org/W2952975409","https://openalex.org/W2953212265","https://openalex.org/W2962780982","https://openalex.org/W2962786581","https://openalex.org/W2962835968","https://openalex.org/W2962844195","https://openalex.org/W2962845550","https://openalex.org/W2962863496","https://openalex.org/W2962911728","https://openalex.org/W2962950660","https://openalex.org/W2962988160","https://openalex.org/W2963228337","https://openalex.org/W2963340555","https://openalex.org/W2963352852","https://openalex.org/W2963367891","https://openalex.org/W2963446712","https://openalex.org/W2963565814","https://openalex.org/W2963778169","https://openalex.org/W2963803379","https://openalex.org/W2963804082","https://openalex.org/W2963807318","https://openalex.org/W2963807877","https://openalex.org/W2963821229","https://openalex.org/W2963840672","https://openalex.org/W2963911037","https://openalex.org/W2963941964","https://openalex.org/W2963964896","https://openalex.org/W2964081807","https://openalex.org/W2964121744","https://openalex.org/W2964286561","https://openalex.org/W2964299589","https://openalex.org/W2972073717","https://openalex.org/W2985108934","https://openalex.org/W2988070836","https://openalex.org/W2998508934","https://openalex.org/W2999905431","https://openalex.org/W2999941859","https://openalex.org/W3029645440","https://openalex.org/W3101036738","https://openalex.org/W3103657382","https://openalex.org/W3118608800","https://openalex.org/W3137092842","https://openalex.org/W3143835353","https://openalex.org/W4205148124","https://openalex.org/W4234552385","https://openalex.org/W4236362309","https://openalex.org/W4237704995","https://openalex.org/W4247950230","https://openalex.org/W4250705075","https://openalex.org/W4255681033","https://openalex.org/W4292363360","https://openalex.org/W4297938497","https://openalex.org/W4380434592"],"related_works":["https://openalex.org/W2974157558","https://openalex.org/W207411050","https://openalex.org/W4387237598","https://openalex.org/W1554644772","https://openalex.org/W2950520577","https://openalex.org/W2494130044","https://openalex.org/W2003935582","https://openalex.org/W3209384898","https://openalex.org/W74409296","https://openalex.org/W1991844655"],"abstract_inverted_index":{"Deep":[0],"Neural":[1],"Networks":[2],"(DNNs)":[3],"are":[4],"becoming":[5],"an":[6],"important":[7],"tool":[8],"in":[9,50,70,78,111],"modern":[10],"computing":[11],"applications.":[12],"Accelerating":[13],"their":[14],"training":[15],"is":[16],"a":[17,38],"major":[18],"challenge":[19],"and":[20,53,63,81,97],"techniques":[21],"range":[22],"from":[23,37,72],"distributed":[24,84,92],"algorithms":[25],"to":[26,83],"low-level":[27],"circuit":[28],"design.":[29],"In":[30],"this":[31],"survey,":[32],"we":[33,105],"describe":[34],"the":[35,54,65,73],"problem":[36],"theoretical":[39],"perspective,":[40],"followed":[41],"by":[42],"approaches":[43],"for":[44,109],"its":[45],"parallelization.":[46],"We":[47,60,87],"present":[48],"trends":[49],"DNN":[51],"architectures":[52],"resulting":[55],"implications":[56],"on":[57,102],"parallelization":[58],"strategies.":[59],"then":[61],"review":[62],"model":[64],"different":[66],"types":[67],"of":[68],"concurrency":[69],"DNNs:":[71],"single":[74],"operator,":[75],"through":[76],"parallelism":[77,110],"network":[79],"inference":[80],"training,":[82],"deep":[85,112],"learning.":[86,113],"discuss":[88],"asynchronous":[89],"stochastic":[90],"optimization,":[91],"system":[93],"architectures,":[94],"communication":[95],"schemes,":[96],"neural":[98],"architecture":[99],"search.":[100],"Based":[101],"those":[103],"approaches,":[104],"extrapolate":[106],"potential":[107],"directions":[108]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":68},{"year":2024,"cited_by_count":73},{"year":2023,"cited_by_count":72},{"year":2022,"cited_by_count":61},{"year":2021,"cited_by_count":113},{"year":2020,"cited_by_count":112},{"year":2019,"cited_by_count":43},{"year":2018,"cited_by_count":10}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}