{"id":"https://openalex.org/W3143293307","doi":"https://doi.org/10.1109/access.2021.3070012","title":"Accelerating Distributed SGD With Group Hybrid Parallelism","display_name":"Accelerating Distributed SGD With Group Hybrid Parallelism","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3143293307","doi":"https://doi.org/10.1109/access.2021.3070012","mag":"3143293307"},"language":"en","primary_location":{"id":"doi:10.1109/access.2021.3070012","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3070012","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09391652.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09391652.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5079698839","display_name":"Kyung-no Joo","orcid":"https://orcid.org/0000-0001-7339-1645"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":true,"raw_author_name":"Kyung-No Joo","raw_affiliation_strings":["School of Electrical Engineering, Korea Advanced Institute of Science and Technology, Daejeon, South Korea"],"raw_orcid":"https://orcid.org/0000-0001-7339-1645","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering, Korea Advanced Institute of Science and Technology, Daejeon, South Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5045403869","display_name":"Chan\u2010Hyun Youn","orcid":"https://orcid.org/0000-0002-3970-7308"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Chan-Hyun Youn","raw_affiliation_strings":["School of Electrical Engineering, Korea Advanced Institute of Science and Technology, Daejeon, South Korea"],"raw_orcid":"https://orcid.org/0000-0002-3970-7308","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering, Korea Advanced Institute of Science and Technology, Daejeon, South Korea","institution_ids":["https://openalex.org/I157485424"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5079698839"],"corresponding_institution_ids":["https://openalex.org/I157485424"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":0.3879,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":{"value":0.59295663,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"9","issue":null,"first_page":"52601","last_page":"52618"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8786566257476807},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7704370021820068},{"id":"https://openalex.org/keywords/data-parallelism","display_name":"Data parallelism","score":0.6896145343780518},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.6471549868583679},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.6272919178009033},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.5861592292785645},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5076013803482056},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.49793362617492676},{"id":"https://openalex.org/keywords/task-parallelism","display_name":"Task parallelism","score":0.48111671209335327},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.43982887268066406},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.43576762080192566},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.3454180955886841},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2795126736164093},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.17843285202980042},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.0857221782207489}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8786566257476807},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7704370021820068},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.6896145343780518},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6471549868583679},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.6272919178009033},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.5861592292785645},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5076013803482056},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.49793362617492676},{"id":"https://openalex.org/C42992933","wikidata":"https://www.wikidata.org/wiki/Q691169","display_name":"Task parallelism","level":3,"score":0.48111671209335327},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.43982887268066406},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.43576762080192566},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3454180955886841},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2795126736164093},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.17843285202980042},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0857221782207489},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2021.3070012","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3070012","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09391652.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:ae8f4659d785456e87859b9661883953","is_oa":true,"landing_page_url":"https://doaj.org/article/ae8f4659d785456e87859b9661883953","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 9, Pp 52601-52618 (2021)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2021.3070012","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3070012","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09391652.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1979152044","display_name":null,"funder_award_id":"20ZS1100","funder_id":"https://openalex.org/F4320322093","funder_display_name":"Electronics and Telecommunications Research Institute"}],"funders":[{"id":"https://openalex.org/F4320322093","display_name":"Electronics and Telecommunications Research Institute","ror":"https://ror.org/03ysstz10"},{"id":"https://openalex.org/F4320323103","display_name":"Agency for Defense Development","ror":"https://ror.org/05fhe0r85"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3143293307.pdf","grobid_xml":"https://content.openalex.org/works/W3143293307.grobid-xml"},"referenced_works_count":81,"referenced_works":["https://openalex.org/W1572016165","https://openalex.org/W1598866093","https://openalex.org/W1652793671","https://openalex.org/W1979566015","https://openalex.org/W2018162709","https://openalex.org/W2059300917","https://openalex.org/W2072566913","https://openalex.org/W2093229192","https://openalex.org/W2120480077","https://openalex.org/W2131613942","https://openalex.org/W2168231600","https://openalex.org/W2287011250","https://openalex.org/W2336650964","https://openalex.org/W2397007421","https://openalex.org/W2489529491","https://openalex.org/W2526749411","https://openalex.org/W2535838896","https://openalex.org/W2541884796","https://openalex.org/W2553581924","https://openalex.org/W2582737986","https://openalex.org/W2617411258","https://openalex.org/W2622263826","https://openalex.org/W2736601468","https://openalex.org/W2752512710","https://openalex.org/W2769644379","https://openalex.org/W2774000609","https://openalex.org/W2785452945","https://openalex.org/W2786066748","https://openalex.org/W2895891814","https://openalex.org/W2907299250","https://openalex.org/W2913570153","https://openalex.org/W2924425593","https://openalex.org/W2942889782","https://openalex.org/W2946270031","https://openalex.org/W2949869455","https://openalex.org/W2952313066","https://openalex.org/W2952926545","https://openalex.org/W2955052486","https://openalex.org/W2955425717","https://openalex.org/W2963339410","https://openalex.org/W2963361660","https://openalex.org/W2963803379","https://openalex.org/W2963959650","https://openalex.org/W2964174152","https://openalex.org/W2969388332","https://openalex.org/W2970155754","https://openalex.org/W2972087877","https://openalex.org/W2991040477","https://openalex.org/W3037847693","https://openalex.org/W3038028469","https://openalex.org/W3105314253","https://openalex.org/W3105668694","https://openalex.org/W4297687186","https://openalex.org/W4298840580","https://openalex.org/W4318619660","https://openalex.org/W6634263594","https://openalex.org/W6635810480","https://openalex.org/W6636806777","https://openalex.org/W6684859321","https://openalex.org/W6696413332","https://openalex.org/W6703420464","https://openalex.org/W6712646927","https://openalex.org/W6728757088","https://openalex.org/W6730009442","https://openalex.org/W6738144653","https://openalex.org/W6739622702","https://openalex.org/W6741002519","https://openalex.org/W6744307745","https://openalex.org/W6746200960","https://openalex.org/W6746839373","https://openalex.org/W6747652958","https://openalex.org/W6747917562","https://openalex.org/W6749257597","https://openalex.org/W6753584990","https://openalex.org/W6756718674","https://openalex.org/W6757290436","https://openalex.org/W6759226220","https://openalex.org/W6760608742","https://openalex.org/W6762718338","https://openalex.org/W6762890684","https://openalex.org/W6763760226"],"related_works":["https://openalex.org/W2950520577","https://openalex.org/W1501159154","https://openalex.org/W2003935582","https://openalex.org/W74409296","https://openalex.org/W1554644772","https://openalex.org/W2494130044","https://openalex.org/W2468095077","https://openalex.org/W29548032","https://openalex.org/W2593878938","https://openalex.org/W305742777"],"abstract_inverted_index":{"The":[0],"scale":[1],"of":[2,28,48,64,129,135,181],"model":[3,143,157,180,185],"parameters":[4],"and":[5,30,69,113,142,149,156,184,187,218],"datasets":[6],"is":[7,32],"rapidly":[8],"growing":[9],"for":[10,38,203],"high":[11],"accuracy":[12,76,102],"in":[13,132,212],"various":[14],"areas.":[15],"To":[16],"train":[17],"a":[18,25,35,126],"large-scale":[19,40,55,137],"deep":[20],"neural":[21],"network":[22,130],"(DNN)":[23],"model,":[24,193],"huge":[26,127],"amount":[27,128],"computation":[29],"memory":[31,153],"required;":[33],"therefore,":[34],"parallelization":[36,201],"technique":[37],"training":[39,98,114,136,166,178,191,205],"DNN":[41,56],"models":[42],"has":[43],"attracted":[44],"attention.":[45],"A":[46],"number":[47],"approaches":[49],"have":[50],"been":[51],"proposed":[52,85],"to":[53,77,95,161],"parallelize":[54],"models,":[57],"but":[58],"these":[59],"schemes":[60],"lack":[61],"scalability":[62],"because":[63],"their":[65],"long":[66],"communication":[67,79,120,147],"time":[68,99,148,167,179,192],"limited":[70],"worker":[71,152],"memory.":[72],"They":[73],"often":[74],"sacrifice":[75],"reduce":[78,146],"time.":[80,206],"In":[81],"this":[82],"work,":[83],"we":[84,194],"an":[86],"efficient":[87],"parallelism":[88,93,144,211,216,226],"strategy":[89,202],"named":[90],"group":[91,209,224],"hybrid":[92,210,225],"(GHP)":[94],"minimize":[96],"the":[97,133,151,165,177,182,190,196,200],"without":[100],"any":[101],"loss.":[103],"Two":[104],"key":[105],"ideas":[106],"inspired":[107],"our":[108,219],"approach.":[109],"First,":[110],"grouping":[111],"workers":[112],"them":[115],"by":[116],"groups":[117],"reduces":[118],"unnecessary":[119],"overhead":[121],"among":[122],"workers.":[123],"It":[124],"saves":[125],"resources":[131],"course":[134],"networks.":[138],"Second,":[139],"mixing":[140],"data":[141,183],"can":[145,168],"mitigate":[150],"issue.":[154],"Data":[155],"paralleism":[158],"are":[159,173],"complementary":[160],"each":[162],"other":[163],"so":[164],"be":[169],"enhanced":[170],"when":[171],"they":[172],"combined.":[174],"We":[175,207],"analyzed":[176],"parallelism,":[186],"based":[188],"on":[189],"demonstrated":[195],"heuristics":[197],"that":[198,223],"determine":[199],"minimizing":[204],"evaluated":[208],"comparison":[213],"with":[214],"existing":[215],"schemes,":[217],"experimental":[220],"results":[221],"show":[222],"outperforms":[227],"them.":[228]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2026-05-06T08:25:59.206177","created_date":"2025-10-10T00:00:00"}