{"id":"https://openalex.org/W3036703963","doi":"https://doi.org/10.1109/tpds.2020.3003307","title":"Distributed Training of Deep Learning Models: A Taxonomic Perspective","display_name":"Distributed Training of Deep Learning Models: A Taxonomic Perspective","publication_year":2020,"publication_date":"2020-06-18","ids":{"openalex":"https://openalex.org/W3036703963","doi":"https://doi.org/10.1109/tpds.2020.3003307","mag":"3036703963"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2020.3003307","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2020.3003307","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2007.03970","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Matthias Langer","orcid":"https://orcid.org/0000-0003-1776-8000"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Matthias Langer","raw_affiliation_strings":["BOSS ZhiPin Career Science Lab (CSL), Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1776-8000","affiliations":[{"raw_affiliation_string":"BOSS ZhiPin Career Science Lab (CSL), Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhen He","orcid":"https://orcid.org/0000-0003-0302-5775"},"institutions":[{"id":"https://openalex.org/I196829312","display_name":"La Trobe University","ror":"https://ror.org/01rxfrp27","country_code":"AU","type":"education","lineage":["https://openalex.org/I196829312"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Zhen He","raw_affiliation_strings":["La Trobe University, Bundoora, Australia"],"raw_orcid":"https://orcid.org/0000-0003-0302-5775","affiliations":[{"raw_affiliation_string":"La Trobe University, Bundoora, Australia","institution_ids":["https://openalex.org/I196829312"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Wenny Rahayu","orcid":"https://orcid.org/0000-0003-2657-4849"},"institutions":[{"id":"https://openalex.org/I196829312","display_name":"La Trobe University","ror":"https://ror.org/01rxfrp27","country_code":"AU","type":"education","lineage":["https://openalex.org/I196829312"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Wenny Rahayu","raw_affiliation_strings":["La Trobe University, Bundoora, Australia"],"raw_orcid":"https://orcid.org/0000-0003-2657-4849","affiliations":[{"raw_affiliation_string":"La Trobe University, Bundoora, Australia","institution_ids":["https://openalex.org/I196829312"]}]},{"author_position":"last","author":{"id":null,"display_name":"Yanbo Xue","orcid":"https://orcid.org/0000-0001-5999-1521"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yanbo Xue","raw_affiliation_strings":["BOSS ZhiPin Career Science Lab (CSL), Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-5999-1521","affiliations":[{"raw_affiliation_string":"BOSS ZhiPin Career Science Lab (CSL), Beijing, China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.2403,"has_fulltext":false,"cited_by_count":93,"citation_normalized_percentile":{"value":0.95276813,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":"31","issue":"12","first_page":"2802","last_page":"2818"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.40299999713897705,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.40299999713897705,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.08399999886751175,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.05570000037550926,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.8163999915122986},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5837000012397766},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.45879998803138733},{"id":"https://openalex.org/keywords/taxonomy","display_name":"Taxonomy (biology)","score":0.40059998631477356},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.39809998869895935},{"id":"https://openalex.org/keywords/cluster","display_name":"Cluster (spacecraft)","score":0.36730000376701355},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.36660000681877136},{"id":"https://openalex.org/keywords/distributed-learning","display_name":"Distributed learning","score":0.34310001134872437}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8737999796867371},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.8163999915122986},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6970999836921692},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5837000012397766},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4964999854564667},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.45879998803138733},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.40059998631477356},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.39809998869895935},{"id":"https://openalex.org/C164866538","wikidata":"https://www.wikidata.org/wiki/Q367351","display_name":"Cluster (spacecraft)","level":2,"score":0.36730000376701355},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.36660000681877136},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.36640000343322754},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.34689998626708984},{"id":"https://openalex.org/C2779582901","wikidata":"https://www.wikidata.org/wiki/Q21013010","display_name":"Distributed learning","level":2,"score":0.34310001134872437},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.3382999897003174},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.32089999318122864},{"id":"https://openalex.org/C97385483","wikidata":"https://www.wikidata.org/wiki/Q16954980","display_name":"Deep belief network","level":3,"score":0.31949999928474426},{"id":"https://openalex.org/C67953723","wikidata":"https://www.wikidata.org/wiki/Q192525","display_name":"Workstation","level":2,"score":0.3125},{"id":"https://openalex.org/C138020889","wikidata":"https://www.wikidata.org/wiki/Q2349659","display_name":"Collaborative learning","level":2,"score":0.3037000000476837},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.29350000619888306},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.28450000286102295},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.27630001306533813},{"id":"https://openalex.org/C3739613","wikidata":"https://www.wikidata.org/wiki/Q679003","display_name":"Distributed Computing Environment","level":2,"score":0.2750999927520752},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.273499995470047},{"id":"https://openalex.org/C70061542","wikidata":"https://www.wikidata.org/wiki/Q989016","display_name":"Distributed database","level":2,"score":0.2685000002384186},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.25690001249313354},{"id":"https://openalex.org/C29140674","wikidata":"https://www.wikidata.org/wiki/Q206637","display_name":"Computer cluster","level":2,"score":0.2531000077724457}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpds.2020.3003307","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2020.3003307","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2007.03970","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2007.03970","pdf_url":"https://arxiv.org/pdf/2007.03970","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2007.03970","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2007.03970","pdf_url":"https://arxiv.org/pdf/2007.03970","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":48,"referenced_works":["https://openalex.org/W2076063813","https://openalex.org/W2117905067","https://openalex.org/W2120480077","https://openalex.org/W2131613942","https://openalex.org/W2194775991","https://openalex.org/W2202546323","https://openalex.org/W2579247884","https://openalex.org/W2755021080","https://openalex.org/W2761434131","https://openalex.org/W2797862701","https://openalex.org/W2802446276","https://openalex.org/W2919115771","https://openalex.org/W2962911728","https://openalex.org/W2963674387","https://openalex.org/W2971624117","https://openalex.org/W2972087877","https://openalex.org/W6604254268","https://openalex.org/W6622473587","https://openalex.org/W6628377381","https://openalex.org/W6631190155","https://openalex.org/W6631564300","https://openalex.org/W6638667902","https://openalex.org/W6638803421","https://openalex.org/W6638926925","https://openalex.org/W6662929504","https://openalex.org/W6679154944","https://openalex.org/W6679393576","https://openalex.org/W6683722107","https://openalex.org/W6684191040","https://openalex.org/W6684859321","https://openalex.org/W6686509673","https://openalex.org/W6687494587","https://openalex.org/W6695752121","https://openalex.org/W6703420464","https://openalex.org/W6713134421","https://openalex.org/W6714387954","https://openalex.org/W6726983090","https://openalex.org/W6730398498","https://openalex.org/W6731098622","https://openalex.org/W6732520560","https://openalex.org/W6738250615","https://openalex.org/W6739554518","https://openalex.org/W6742061381","https://openalex.org/W6756718674","https://openalex.org/W6764309476","https://openalex.org/W6767026093","https://openalex.org/W6769062451","https://openalex.org/W6889677906"],"related_works":[],"abstract_inverted_index":{"Distributed":[0],"deep":[1,6,41,49,108,125,185],"learning":[2,126,186],"systems":[3],"(DDLS)":[4],"train":[5,74],"neural":[7,50,109],"network":[8,51],"models":[9,76,127],"by":[10,117,155],"utilizing":[11],"the":[12,43,56,99,119,149,165],"distributed":[13,134,184],"resources":[14],"of":[15,19,39,46,66,114,148,183],"a":[16,112,136,181],"cluster.":[17],"Developers":[18],"DDLS":[20,67,79,157],"are":[21,103,153],"required":[22],"to":[23,27,68,73,84,94,138],"make":[24],"many":[25],"decisions":[26],"process":[28],"their":[29,33,85,160],"particular":[30],"workloads":[31,131],"in":[32,53,61,71,111,135],"chosen":[34],"environment":[35],"efficiently.":[36],"The":[37],"advent":[38],"GPU-based":[40],"learning,":[42],"ever-increasing":[44],"size":[45],"datasets,":[47],"and":[48,89,128,158,162,170],"models,":[52],"combination":[54],"with":[55,123],"bandwidth":[57],"constraints":[58],"that":[59,102,152],"exist":[60],"cluster":[62,113,137],"environments":[63],"require":[64],"developers":[65],"be":[69,133],"innovative":[70],"order":[72],"high-quality":[75],"quickly.":[77],"Comparing":[78],"side-by-side":[80],"is":[81],"difficult":[82],"due":[83],"extensive":[86],"feature":[87],"lists":[88],"architectural":[90],"deviations.":[91],"We":[92],"aim":[93],"shine":[95],"some":[96],"light":[97],"on":[98,164],"fundamental":[100],"principles":[101],"at":[104],"work":[105],"when":[106],"training":[107,124,166],"networks":[110],"independent":[115],"machines":[116],"analyzing":[118],"general":[120],"properties":[121],"associated":[122],"how":[129],"such":[130],"can":[132],"achieve":[139],"collaborative":[140],"model":[141],"training.":[142],"Thereby":[143],"we":[144,173],"provide":[145],"an":[146],"overview":[147],"different":[150,175],"techniques":[151,176],"used":[154],"contemporary":[156],"discuss":[159],"influence":[161],"implications":[163],"process.":[167],"To":[168],"conceptualize":[169],"compare":[171],"DDLS,":[172],"group":[174],"into":[177],"categories,":[178],"thus":[179],"establishing":[180],"taxonomy":[182],"systems.":[187]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":21},{"year":2024,"cited_by_count":21},{"year":2023,"cited_by_count":17},{"year":2022,"cited_by_count":14},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":1}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2020-06-25T00:00:00"}