{"id":"https://openalex.org/W4411800445","doi":"https://doi.org/10.1109/ccgrid64434.2025.00044","title":"An Optimization Technique for Hiding Communication Costs in 3D Parallel Training of Deep Learning","display_name":"An Optimization Technique for Hiding Communication Costs in 3D Parallel Training of Deep Learning","publication_year":2025,"publication_date":"2025-05-19","ids":{"openalex":"https://openalex.org/W4411800445","doi":"https://doi.org/10.1109/ccgrid64434.2025.00044"},"language":"en","primary_location":{"id":"doi:10.1109/ccgrid64434.2025.00044","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ccgrid64434.2025.00044","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 25th International Symposium on Cluster, Cloud and Internet Computing (CCGrid)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5093752369","display_name":"Ryubu Hosoki","orcid":"https://orcid.org/0000-0002-8435-7829"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ryubu Hosoki","raw_affiliation_strings":["Institute of Science Tokyo,Yokohama,Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Science Tokyo,Yokohama,Japan","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025610878","display_name":"Kento Sato","orcid":"https://orcid.org/0000-0001-7850-2121"},"institutions":[{"id":"https://openalex.org/I4210129730","display_name":"RIKEN Center for Computational Science","ror":"https://ror.org/03r519674","country_code":"JP","type":"facility","lineage":["https://openalex.org/I4210110652","https://openalex.org/I4210129730"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Kento Sato","raw_affiliation_strings":["RIKEN Center for Computational Science,Kobe,Japan","RIKEN CCS - RIKEN Center for Computational Science [Kobe] (7 Chome-1-26 Minatojima Minamimachi, Chuo Ward, Kobe, Hyogo 650-0047, Japon - Japan)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RIKEN Center for Computational Science,Kobe,Japan","institution_ids":["https://openalex.org/I4210129730"]},{"raw_affiliation_string":"RIKEN CCS - RIKEN Center for Computational Science [Kobe] (7 Chome-1-26 Minatojima Minamimachi, Chuo Ward, Kobe, Hyogo 650-0047, Japon - Japan)","institution_ids":["https://openalex.org/I4210129730"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011254074","display_name":"Toshio Endo","orcid":"https://orcid.org/0000-0001-7297-6211"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Toshio Endo","raw_affiliation_strings":["Institute of Science Tokyo,Yokohama,Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Science Tokyo,Yokohama,Japan","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073645491","display_name":"Julien Bigot","orcid":"https://orcid.org/0000-0002-0015-4304"},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I195731000","display_name":"Universit\u00e9 de Versailles Saint-Quentin-en-Yvelines","ror":"https://ror.org/03mkjjy25","country_code":"FR","type":"education","lineage":["https://openalex.org/I195731000","https://openalex.org/I277688954"]},{"id":"https://openalex.org/I2738703131","display_name":"Commissariat \u00e0 l'\u00c9nergie Atomique et aux \u00c9nergies Alternatives","ror":"https://ror.org/00jjx8s55","country_code":"FR","type":"government","lineage":["https://openalex.org/I2738703131"]},{"id":"https://openalex.org/I4210125654","display_name":"Maison de la Simulation","ror":"https://ror.org/03jv6w209","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I195731000","https://openalex.org/I2738703131","https://openalex.org/I2738703131","https://openalex.org/I277688954","https://openalex.org/I277688954","https://openalex.org/I4210113668","https://openalex.org/I4210125654","https://openalex.org/I4210159245"]},{"id":"https://openalex.org/I4210128565","display_name":"CEA Paris-Saclay","ror":"https://ror.org/03n15ch10","country_code":"FR","type":"government","lineage":["https://openalex.org/I2738703131","https://openalex.org/I277688954","https://openalex.org/I4210128565"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Julien Bigot","raw_affiliation_strings":["Université Paris-Saclay, UVSQ, CNRS, CEA, Maison de la Simulation,Gif-sur-Yvette,France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Université Paris-Saclay, UVSQ, CNRS, CEA, Maison de la Simulation,Gif-sur-Yvette,France","institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I195731000","https://openalex.org/I2738703131","https://openalex.org/I4210125654","https://openalex.org/I4210128565"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008200333","display_name":"E. Audit","orcid":"https://orcid.org/0000-0002-4960-5372"},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I195731000","display_name":"Universit\u00e9 de Versailles Saint-Quentin-en-Yvelines","ror":"https://ror.org/03mkjjy25","country_code":"FR","type":"education","lineage":["https://openalex.org/I195731000","https://openalex.org/I277688954"]},{"id":"https://openalex.org/I2738703131","display_name":"Commissariat \u00e0 l'\u00c9nergie Atomique et aux \u00c9nergies Alternatives","ror":"https://ror.org/00jjx8s55","country_code":"FR","type":"government","lineage":["https://openalex.org/I2738703131"]},{"id":"https://openalex.org/I4210125654","display_name":"Maison de la Simulation","ror":"https://ror.org/03jv6w209","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I195731000","https://openalex.org/I2738703131","https://openalex.org/I2738703131","https://openalex.org/I277688954","https://openalex.org/I277688954","https://openalex.org/I4210113668","https://openalex.org/I4210125654","https://openalex.org/I4210159245"]},{"id":"https://openalex.org/I4210128565","display_name":"CEA Paris-Saclay","ror":"https://ror.org/03n15ch10","country_code":"FR","type":"government","lineage":["https://openalex.org/I2738703131","https://openalex.org/I277688954","https://openalex.org/I4210128565"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Edouard Audit","raw_affiliation_strings":["Université Paris-Saclay, UVSQ, CNRS, CEA, Maison de la Simulation,Gif-sur-Yvette,France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Université Paris-Saclay, UVSQ, CNRS, CEA, Maison de la Simulation,Gif-sur-Yvette,France","institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I195731000","https://openalex.org/I2738703131","https://openalex.org/I4210125654","https://openalex.org/I4210128565"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.13735651,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"472","last_page":"481"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.993399977684021,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7964720129966736},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.6224024295806885},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.537534773349762},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.5263872146606445},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.34890007972717285},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3342122435569763},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.334006667137146}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7964720129966736},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6224024295806885},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.537534773349762},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5263872146606445},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.34890007972717285},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3342122435569763},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.334006667137146},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/ccgrid64434.2025.00044","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ccgrid64434.2025.00044","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 25th International Symposium on Cluster, Cloud and Internet Computing (CCGrid)","raw_type":"proceedings-article"},{"id":"pmh:oai:HAL:hal-05299979v1","is_oa":false,"landing_page_url":"https://hal.science/hal-05299979","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Unknown, May 2025, Unknown, Unknown Region. pp.1-10, ⟨10.1109/ccgrid64434.2025.00044⟩","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W2117539524","https://openalex.org/W2899971035","https://openalex.org/W2948902769","https://openalex.org/W2969388332","https://openalex.org/W2973727699","https://openalex.org/W2991040477","https://openalex.org/W3019389960","https://openalex.org/W3081168214","https://openalex.org/W3101860695","https://openalex.org/W3129831491","https://openalex.org/W3163465001","https://openalex.org/W3167220634","https://openalex.org/W3171087525","https://openalex.org/W3175937250","https://openalex.org/W3204538018","https://openalex.org/W3204998121","https://openalex.org/W3206832494","https://openalex.org/W3210871626","https://openalex.org/W4221167110","https://openalex.org/W4312349930","https://openalex.org/W4327694855","https://openalex.org/W4389326242"],"related_works":["https://openalex.org/W230091440","https://openalex.org/W2233261550","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W2997094352","https://openalex.org/W4375867731","https://openalex.org/W3216976533","https://openalex.org/W100620283","https://openalex.org/W2495260952","https://openalex.org/W4380075502"],"abstract_inverted_index":{"In":[0],"large-scale":[1],"model":[2],"training,":[3],"distributing":[4],"the":[5,70,91,101,128],"workload":[6],"is":[7],"essential":[8],"to":[9,63,123,141],"address":[10,111],"memory":[11],"usage":[12],"and":[13,25,42,66],"computation":[14,125],"time":[15],"constraints.":[16],"3D":[17,73,88],"parallelism,":[18,22,24,27,74],"which":[19,118],"combines":[20],"data":[21],"tensor":[23],"pipeline":[26,79],"has":[28],"emerged":[29],"as":[30],"a":[31,46,115,142],"state-of-the-art":[32],"method":[33],"for":[34,78],"distributed":[35],"training.":[36,148],"However,":[37],"exploring":[38],"optimal":[39],"parallel":[40],"strategies":[41],"modifying":[43],"code":[44],"places":[45],"significant":[47],"burden":[48],"on":[49],"users.":[50],"While":[51],"some":[52],"machine":[53,93],"learning":[54,94],"compilers":[55],"support":[56,69],"automatic":[57],"parallelization,":[58],"they":[59],"are":[60],"generally":[61],"limited":[62],"SPMD":[64],"parallelism":[65,77,89],"do":[67],"not":[68],"automation":[71],"of":[72,103,130],"including":[75],"MPMD":[76],"parallelism.":[80],"This":[81,134],"study":[82],"identifies":[83],"key":[84],"issues":[85],"in":[86,106,146],"implementing":[87],"with":[90],"XLA":[92],"compiler,":[95],"particularly":[96],"communication":[97,121,131],"delays":[98],"caused":[99],"by":[100],"limitations":[102],"sharding":[104],"notation":[105],"XLA's":[107],"intermediate":[108],"representation.":[109],"To":[110],"this,":[112],"we":[113],"propose":[114],"commshift":[116],"optimization,":[117],"relocates":[119],"specific":[120],"instructions":[122],"different":[124],"phases,":[126],"reducing":[127],"impact":[129],"waiting":[132],"times.":[133],"optimization":[135],"reduces":[136],"training":[137],"time,":[138],"achieving":[139],"up":[140],"27%":[143],"throughput":[144],"improvement":[145],"GPT-J":[147]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}