{"id":"https://openalex.org/W4416004435","doi":"https://doi.org/10.1145/3731599.3767571","title":"Scalable Neural Network Training: Distributed Data-Parallel Approaches","display_name":"Scalable Neural Network Training: Distributed Data-Parallel Approaches","publication_year":2025,"publication_date":"2025-11-07","ids":{"openalex":"https://openalex.org/W4416004435","doi":"https://doi.org/10.1145/3731599.3767571"},"language":null,"primary_location":{"id":"doi:10.1145/3731599.3767571","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3731599.3767571","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the SC '25 Workshops of the International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092682778","display_name":"Fernando V\u00e1zquez-Novoa","orcid":"https://orcid.org/0000-0001-5634-509X"},"institutions":[{"id":"https://openalex.org/I2799803557","display_name":"Barcelona Supercomputing Center","ror":"https://ror.org/05sd8tv96","country_code":"ES","type":"facility","lineage":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]},{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Fernando V\u00e1zquez-Novoa","raw_affiliation_strings":["Barcelona Supercomputing Center (BSC), Barcelona, Spain"],"raw_orcid":"https://orcid.org/0000-0001-5634-509X","affiliations":[{"raw_affiliation_string":"Barcelona Supercomputing Center (BSC), Barcelona, Spain","institution_ids":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101612028","display_name":"Pedro L\u00f3pez","orcid":"https://orcid.org/0000-0003-4544-955X"},"institutions":[{"id":"https://openalex.org/I60053951","display_name":"Universitat Polit\u00e8cnica de Val\u00e8ncia","ror":"https://ror.org/01460j859","country_code":"ES","type":"education","lineage":["https://openalex.org/I60053951"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Pedro L\u00f3pez","raw_affiliation_strings":["Universidad Politecnica de Valencia, Valencia, Spain"],"raw_orcid":"https://orcid.org/0000-0003-4544-955X","affiliations":[{"raw_affiliation_string":"Universidad Politecnica de Valencia, Valencia, Spain","institution_ids":["https://openalex.org/I60053951"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046815809","display_name":"Jos\u00e9 Flich","orcid":"https://orcid.org/0000-0001-8581-6284"},"institutions":[{"id":"https://openalex.org/I60053951","display_name":"Universitat Polit\u00e8cnica de Val\u00e8ncia","ror":"https://ror.org/01460j859","country_code":"ES","type":"education","lineage":["https://openalex.org/I60053951"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Jos\u00e9 Flich","raw_affiliation_strings":["Universidad Politecnica de Valencia, Valencia, Spain"],"raw_orcid":"https://orcid.org/0000-0001-8581-6284","affiliations":[{"raw_affiliation_string":"Universidad Politecnica de Valencia, Valencia, Spain","institution_ids":["https://openalex.org/I60053951"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015588225","display_name":"Rosa M. Badia","orcid":"https://orcid.org/0000-0003-2941-5499"},"institutions":[{"id":"https://openalex.org/I2799803557","display_name":"Barcelona Supercomputing Center","ror":"https://ror.org/05sd8tv96","country_code":"ES","type":"facility","lineage":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]},{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Rosa M. Badia","raw_affiliation_strings":["Barcelona Supercomputing Center (BSC), Barcelona, Spain"],"raw_orcid":"https://orcid.org/0000-0003-2941-5499","affiliations":[{"raw_affiliation_string":"Barcelona Supercomputing Center (BSC), Barcelona, Spain","institution_ids":["https://openalex.org/I2799803557","https://openalex.org/I9617848"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2088","last_page":"2099"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.18240000307559967,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.18240000307559967,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.15369999408721924,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12303","display_name":"Tensor decomposition and applications","score":0.09109999984502792,"subfield":{"id":"https://openalex.org/subfields/2605","display_name":"Computational Mathematics"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7766000032424927},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.6834999918937683},{"id":"https://openalex.org/keywords/abstraction","display_name":"Abstraction","score":0.6660000085830688},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.620199978351593},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5983999967575073},{"id":"https://openalex.org/keywords/limit","display_name":"Limit (mathematics)","score":0.47609999775886536}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8309000134468079},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7766000032424927},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.6834999918937683},{"id":"https://openalex.org/C124304363","wikidata":"https://www.wikidata.org/wiki/Q673661","display_name":"Abstraction","level":2,"score":0.6660000085830688},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.620199978351593},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5983999967575073},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.570900022983551},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.47609999775886536},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4724000096321106},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4250999987125397},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.382099986076355},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.35659998655319214},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.31869998574256897},{"id":"https://openalex.org/C108734733","wikidata":"https://www.wikidata.org/wiki/Q1172333","display_name":"Data synchronization","level":3,"score":0.3149000108242035},{"id":"https://openalex.org/C155281189","wikidata":"https://www.wikidata.org/wiki/Q3518150","display_name":"Tensor (intrinsic definition)","level":2,"score":0.2980000078678131},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.29190000891685486},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2531999945640564}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3731599.3767571","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3731599.3767571","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the SC '25 Workshops of the International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6457506651","display_name":null,"funder_award_id":"PRE2022-104134, CEX2021-001148-S, and PID2023-147979NB-C2","funder_id":"https://openalex.org/F4320315062","funder_display_name":"Ministerio de Ciencia, Innovaci\u00f3n y Universidades"}],"funders":[{"id":"https://openalex.org/F4320315062","display_name":"Ministerio de Ciencia, Innovaci\u00f3n y Universidades","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W1168671556","https://openalex.org/W1598866093","https://openalex.org/W1994616650","https://openalex.org/W2004206314","https://openalex.org/W2294929133","https://openalex.org/W2792287754","https://openalex.org/W2809254203","https://openalex.org/W2995053868","https://openalex.org/W3010913348","https://openalex.org/W3086105743","https://openalex.org/W4229801112","https://openalex.org/W4233045210","https://openalex.org/W4293775467","https://openalex.org/W4301239768","https://openalex.org/W4306955484","https://openalex.org/W4394639701","https://openalex.org/W4405240720","https://openalex.org/W4406779522"],"related_works":[],"abstract_inverted_index":{"Training":[0],"large":[1],"neural":[2],"networks":[3],"is":[4],"computationally":[5],"demanding":[6],"and":[7,53,72],"often":[8],"limited":[9],"by":[10,50,77],"synchronization":[11,49],"overhead":[12],"in":[13],"distributed":[14,80],"environments.":[15],"Traditional":[16],"data-parallel":[17,45],"frameworks,":[18],"such":[19],"as":[20],"Horovod":[21],"or":[22],"PyTorch":[23],"DDP,":[24],"average":[25],"gradients":[26],"at":[27,56],"every":[28],"batch,":[29],"which":[30],"can":[31],"limit":[32],"scalability":[33],"due":[34],"to":[35],"communication":[36],"bottlenecks.":[37],"In":[38],"this":[39],"work,":[40],"we":[41],"propose":[42],"two":[43],"novel":[44],"strategies":[46],"that":[47,84],"reduce":[48],"averaging":[51],"weights":[52],"biases":[54],"only":[55],"the":[57,67],"end":[58],"of":[59],"each":[60],"epoch.":[61],"These":[62],"methods":[63],"are":[64],"implemented":[65],"using":[66],"PyCOMPSs":[68],"task-based":[69],"programming":[70],"model":[71],"integrated":[73],"into":[74],"dislib,":[75],"enabled":[76],"a":[78],"new":[79],"tensor":[81],"abstraction":[82],"(ds-tensor)":[83],"supports":[85],"multidimensional":[86],"data":[87],"structures":[88],"suitable":[89],"for":[90],"deep":[91],"learning":[92],"workloads.":[93]},"counts_by_year":[],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-11-07T00:00:00"}