{"id":"https://openalex.org/W4316252369","doi":"https://doi.org/10.1145/3545008.3545024","title":"HSP: Hybrid Synchronous Parallelism for Fast Distributed Deep Learning","display_name":"HSP: Hybrid Synchronous Parallelism for Fast Distributed Deep Learning","publication_year":2022,"publication_date":"2022-08-29","ids":{"openalex":"https://openalex.org/W4316252369","doi":"https://doi.org/10.1145/3545008.3545024"},"language":"en","primary_location":{"id":"doi:10.1145/3545008.3545024","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3545008.3545024","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 51st International Conference on Parallel Processing","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100447650","display_name":"Yijun Li","orcid":"https://orcid.org/0000-0003-4768-8087"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yijun Li","raw_affiliation_strings":["Central South University, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University, China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008483730","display_name":"Jiawei Huang","orcid":"https://orcid.org/0000-0002-7578-4490"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiawei Huang","raw_affiliation_strings":["Central South University, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University, China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014213647","display_name":"Zhaoyi Li","orcid":"https://orcid.org/0000-0002-9677-2368"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhaoyi Li","raw_affiliation_strings":["Central South University, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University, China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102654693","display_name":"Shengwen Zhou","orcid":null},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shengwen Zhou","raw_affiliation_strings":["Central South University, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University, China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027239963","display_name":"Wanchun Jiang","orcid":"https://orcid.org/0000-0001-5067-321X"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wanchun Jiang","raw_affiliation_strings":["Central South University, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University, China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100438360","display_name":"Jianxin Wang","orcid":"https://orcid.org/0000-0003-1516-0480"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianxin Wang","raw_affiliation_strings":["Central South University, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Central South University, China","institution_ids":["https://openalex.org/I139660479"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I139660479"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/asynchronous-communication","display_name":"Asynchronous communication","score":0.8906553983688354},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8264201879501343},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.7908065319061279},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.7025900483131409},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.6579201221466064},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.5742453336715698},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.5424713492393494},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.4442393481731415},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4165022671222687},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.31050366163253784},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.24145832657814026},{"id":"https://openalex.org/keywords/channel","display_name":"Channel (broadcasting)","score":0.23289012908935547}],"concepts":[{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.8906553983688354},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8264201879501343},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.7908065319061279},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.7025900483131409},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.6579201221466064},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.5742453336715698},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5424713492393494},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.4442393481731415},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4165022671222687},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31050366163253784},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.24145832657814026},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.23289012908935547},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3545008.3545024","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3545008.3545024","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 51st International Conference on Parallel Processing","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.46000000834465027}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W1442374986","https://openalex.org/W2060393849","https://openalex.org/W2127941149","https://openalex.org/W2132737349","https://openalex.org/W2160815625","https://openalex.org/W2168231600","https://openalex.org/W2173213060","https://openalex.org/W2194775991","https://openalex.org/W2612026221","https://openalex.org/W2618530766","https://openalex.org/W2626721309","https://openalex.org/W2807778630","https://openalex.org/W2894576937","https://openalex.org/W2920397365","https://openalex.org/W2944793600","https://openalex.org/W2963433607","https://openalex.org/W2963748441","https://openalex.org/W2975712713","https://openalex.org/W2977528949","https://openalex.org/W2982664135","https://openalex.org/W3047002910","https://openalex.org/W3080451848","https://openalex.org/W3106914464","https://openalex.org/W3109982791","https://openalex.org/W3118608800","https://openalex.org/W3155611867","https://openalex.org/W3155848860","https://openalex.org/W6683738474","https://openalex.org/W6684859321"],"related_works":["https://openalex.org/W2058965144","https://openalex.org/W2164382479","https://openalex.org/W2146343568","https://openalex.org/W98480971","https://openalex.org/W4246549241","https://openalex.org/W2168758875","https://openalex.org/W2410733619","https://openalex.org/W2963483475","https://openalex.org/W207411050","https://openalex.org/W2120877146"],"abstract_inverted_index":{"In":[0,59],"the":[1,7,12,38,46,70,80,113,141],"parameter-server-based":[2],"distributed":[3],"deep":[4,130,147],"learning":[5,131,148],"system,":[6],"workers":[8,87,105],"simultaneously":[9],"communicate":[10],"with":[11,108],"parameter":[13,39,81],"server":[14,82],"to":[15,36,45,88,106,111,152],"refine":[16],"model":[17],"parameters,":[18],"easily":[19],"resulting":[20],"in":[21],"severe":[22],"network":[23,90],"contention.":[24],"To":[25],"solve":[26],"this":[27,60],"problem,":[28],"Asynchronous":[29],"Parallel":[30],"(ASP)":[31],"strategy":[32],"enables":[33],"each":[34,98],"worker":[35],"update":[37],"independently":[40],"without":[41,73],"synchronization.":[42],"However,":[43],"due":[44],"inconsistency":[47],"of":[48,76,115,122,144],"parameters":[49,96],"among":[50],"workers,":[51],"ASP":[52],"experiences":[53],"accuracy":[54],"loss":[55],"and":[56,92,125],"slow":[57],"convergence.":[58],"paper,":[61],"we":[62],"propose":[63],"Hybrid":[64],"Synchronous":[65],"Parallelism":[66],"(HSP),":[67],"which":[68],"mitigates":[69],"communication":[71],"contention":[72],"excessive":[74],"degradation":[75],"convergence":[77,123,142],"speed.":[78],"Specifically,":[79],"sequentially":[83],"pulls":[84],"gradients":[85],"from":[86],"eliminate":[89],"congestion":[91],"synchronizes":[93],"all":[94],"up-to-date":[95],"after":[97],"iteration.":[99],"Meanwhile,":[100],"HSP":[101,127,139],"cautiously":[102],"lets":[103],"idle":[104],"compute":[107],"out-of-date":[109],"weights":[110],"maximize":[112],"utilizations":[114],"computing":[116],"resources.":[117],"We":[118],"provide":[119],"theoretical":[120],"analysis":[121],"efficiency":[124],"implement":[126],"on":[128],"popular":[129],"(DL)":[132],"framework.":[133],"The":[134],"test":[135],"results":[136],"show":[137],"that":[138],"improves":[140],"speedup":[143],"three":[145],"classical":[146],"models":[149],"by":[150],"up":[151],"67%.":[153]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}