{"id":"https://openalex.org/W3134742951","doi":"https://doi.org/10.1109/tcc.2021.3062398","title":"GSSP: Eliminating Stragglers Through Grouping Synchronous for Distributed Deep Learning in Heterogeneous Cluster","display_name":"GSSP: Eliminating Stragglers Through Grouping Synchronous for Distributed Deep Learning in Heterogeneous Cluster","publication_year":2021,"publication_date":"2021-02-26","ids":{"openalex":"https://openalex.org/W3134742951","doi":"https://doi.org/10.1109/tcc.2021.3062398","mag":"3134742951"},"language":"en","primary_location":{"id":"doi:10.1109/tcc.2021.3062398","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcc.2021.3062398","pdf_url":null,"source":{"id":"https://openalex.org/S2492498579","display_name":"IEEE Transactions on Cloud Computing","issn_l":"2168-7161","issn":["2168-7161","2372-0018"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cloud Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008194128","display_name":"Haifeng Sun","orcid":"https://orcid.org/0000-0003-3072-7422"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haifeng Sun","raw_affiliation_strings":["State Key Laboratory of Networking and Switching Technology, Beijing University of Posts and Telecommunications, Beijing, China","EBUPT.COM, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3072-7422","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Networking and Switching Technology, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]},{"raw_affiliation_string":"EBUPT.COM, Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064127208","display_name":"Zhiyi Gui","orcid":null},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiyi Gui","raw_affiliation_strings":["State Key Laboratory of Networking and Switching Technology, Beijing University of Posts and Telecommunications, Beijing, China","EBUPT.COM, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Networking and Switching Technology, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]},{"raw_affiliation_string":"EBUPT.COM, Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043464306","display_name":"Song Guo","orcid":"https://orcid.org/0000-0001-9831-2202"},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Song Guo","raw_affiliation_strings":["Department of Computing, Hong Kong Polytechnic University, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0001-9831-2202","affiliations":[{"raw_affiliation_string":"Department of Computing, Hong Kong Polytechnic University, Hong Kong","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100406584","display_name":"Qi Qi","orcid":"https://orcid.org/0000-0003-0829-4624"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qi Qi","raw_affiliation_strings":["State Key Laboratory of Networking and Switching Technology, Beijing University of Posts and Telecommunications, Beijing, China","EBUPT.COM, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-0829-4624","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Networking and Switching Technology, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]},{"raw_affiliation_string":"EBUPT.COM, Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100432460","display_name":"Jingyu Wang","orcid":"https://orcid.org/0000-0002-2182-2228"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jingyu Wang","raw_affiliation_strings":["State Key Laboratory of Networking and Switching Technology, Beijing University of Posts and Telecommunications, Beijing, China","EBUPT.COM, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-2182-2228","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Networking and Switching Technology, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]},{"raw_affiliation_string":"EBUPT.COM, Beijing, China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5055685073","display_name":"Jianxin Liao","orcid":"https://orcid.org/0000-0003-1486-0573"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianxin Liao","raw_affiliation_strings":["State Key Laboratory of Networking and Switching Technology, Beijing University of Posts and Telecommunications, Beijing, China","EBUPT.COM, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1486-0573","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Networking and Switching Technology, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]},{"raw_affiliation_string":"EBUPT.COM, Beijing, China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.1809,"has_fulltext":false,"cited_by_count":19,"citation_normalized_percentile":{"value":0.7782602,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"10","issue":"4","first_page":"2637","last_page":"2648"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8177316188812256},{"id":"https://openalex.org/keywords/server","display_name":"Server","score":0.5188775658607483},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.46054762601852417},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.3350735306739807},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.28594955801963806}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8177316188812256},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.5188775658607483},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.46054762601852417},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3350735306739807},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.28594955801963806}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tcc.2021.3062398","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcc.2021.3062398","pdf_url":null,"source":{"id":"https://openalex.org/S2492498579","display_name":"IEEE Transactions on Cloud Computing","issn_l":"2168-7161","issn":["2168-7161","2372-0018"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cloud Computing","raw_type":"journal-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-129415","is_oa":false,"landing_page_url":"http://lbdiscover.ust.hk/uresolver?url_ver=Z39.88-2004&rft_val_fmt=info:ofi/fmt:kev:mtx:journal&rfr_id=info:sid/HKUST:SPI&rft.genre=article&rft.issn=2168-7161&rft.volume=10&rft.issue=4&rft.date=2022&rft.spage=2637&rft.aulast=Sun&rft.aufirst=Haifeng&rft.atitle=GSSP%3A+Eliminating+Stragglers+Through+Grouping+Synchronous+for+Distributed+Deep+Learning+in+Heterogeneous+Cluster&rft.title=IEEE+Transactions+on+Cloud+Computing","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4255581185","display_name":"\u9762\u5411\u7f51\u8054\u81ea\u52a8\u9a7e\u9a76\u7684\u6682\u6001\u8fb9\u7f18\u8d44\u6e90\u534f\u4f5c\u673a\u7406\u7814\u7a76","funder_award_id":"62071067","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5092157715","display_name":"\u57fa\u4e8e\u4eba\u5de5\u667a\u80fd\u7684\u8f6f\u4ef6\u5b9a\u4e49\u7f51\u7edc\u6d41\u91cf\u8bc6\u522b\u4e0e\u8d44\u6e90\u5206\u914d\u7814\u7a76","funder_award_id":"61771068","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7111805306","display_name":"\u9762\u5411\u8fb9\u7f18\u667a\u80fd\u7684\u8d44\u6e90\u914d\u7f6e\u4e0e\u90e8\u7f72\u4f18\u5316\u5173\u952e\u6280\u672f\u7814\u7a76","funder_award_id":"61872310","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":60,"referenced_works":["https://openalex.org/W1833917188","https://openalex.org/W1975934607","https://openalex.org/W1980287119","https://openalex.org/W2022508996","https://openalex.org/W2060393849","https://openalex.org/W2130062883","https://openalex.org/W2130942839","https://openalex.org/W2132737349","https://openalex.org/W2138243089","https://openalex.org/W2168231600","https://openalex.org/W2194775991","https://openalex.org/W2336650964","https://openalex.org/W2339765813","https://openalex.org/W2405578611","https://openalex.org/W2407022425","https://openalex.org/W2606891064","https://openalex.org/W2612026221","https://openalex.org/W2617766261","https://openalex.org/W2618530766","https://openalex.org/W2626580042","https://openalex.org/W2750786826","https://openalex.org/W2769644379","https://openalex.org/W2774000609","https://openalex.org/W2794365787","https://openalex.org/W2899771611","https://openalex.org/W2919115771","https://openalex.org/W2920397365","https://openalex.org/W2952230511","https://openalex.org/W2964923275","https://openalex.org/W2965481041","https://openalex.org/W2978015420","https://openalex.org/W2982408761","https://openalex.org/W2982475424","https://openalex.org/W2982664135","https://openalex.org/W3003029937","https://openalex.org/W3009014747","https://openalex.org/W3011400958","https://openalex.org/W3101036738","https://openalex.org/W3109982791","https://openalex.org/W3131992802","https://openalex.org/W4295830359","https://openalex.org/W6638533276","https://openalex.org/W6665801690","https://openalex.org/W6679314259","https://openalex.org/W6679393576","https://openalex.org/W6679436768","https://openalex.org/W6679660245","https://openalex.org/W6680402377","https://openalex.org/W6683738474","https://openalex.org/W6684859321","https://openalex.org/W6703420464","https://openalex.org/W6738460352","https://openalex.org/W6739554518","https://openalex.org/W6746200960","https://openalex.org/W6746839373","https://openalex.org/W6747481501","https://openalex.org/W6749740488","https://openalex.org/W6756040250","https://openalex.org/W6773716851","https://openalex.org/W6775047284"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2092530219","https://openalex.org/W2388464034","https://openalex.org/W2533125852","https://openalex.org/W2140460949","https://openalex.org/W2105580438","https://openalex.org/W2057435755","https://openalex.org/W2390279801","https://openalex.org/W2358668433"],"abstract_inverted_index":{"Distributed":[0],"deep":[1,9],"learning":[2],"has":[3],"been":[4],"widely":[5],"used":[6],"in":[7],"training":[8,28,69,161],"neural":[10],"networks,":[11],"especially":[12],"for":[13,50,123,176],"big":[14],"models":[15],"on":[16,54,143,145,149,153],"massive":[17],"datasets.":[18],"Parameter":[19],"Server":[20],"(PS)":[21],"architecture":[22],"is":[23],"the":[24,34,67,108,120,160,179],"most":[25],"popular":[26],"distributed":[27,52],"framework,":[29],"which":[30,64,80,117],"can":[31,173],"flexibly":[32],"design":[33],"global":[35],"parameter":[36],"update":[37],"manner.":[38],"However,":[39],"when":[40],"scaling":[41],"to":[42,57,103,128],"complex":[43],"heterogeneous":[44],"clusters,":[45],"stragglers":[46,105],"make":[47,129,174],"it":[48],"difficult":[49],"existing":[51],"paradigms":[53],"PS":[55],"framework":[56],"balance":[58],"between":[59],"synchronous":[60],"waiting":[61],"and":[62,106,133,151],"staleness,":[63],"slows":[65],"down":[66],"model":[68,109],"sharply.":[70],"In":[71],"this":[72],"article,":[73],"we":[74],"proposeGrouping":[75,113],"Stale":[76,93],"Synchronous":[77,94],"Parallel(GSSP)":[78],"scheme,":[79],"groups":[81],"workers":[82,91],"with":[83,99,164],"similar":[84],"performance":[85],"together.":[86],"Group":[87],"servers":[88],"coordinate":[89],"intra-group":[90],"using":[92],"Parallel":[95],"while":[96],"they":[97],"communicate":[98],"each":[100,124],"other":[101],"asynchronously":[102],"eliminate":[104],"refine":[107],"weights.":[110],"We":[111,139],"further":[112],"Dynamic":[114],"Tok-K":[115],"Sparsification(GDTopK),":[116],"dynamically":[118],"adjusts":[119],"upload":[121],"ratio":[122],"group":[125],"so":[126],"as":[127],"communication":[130],"volume":[131],"differentiated":[132],"mitigate":[134],"inter-group":[135],"iteration":[136],"speed":[137],"gap.":[138],"have":[140],"conducted":[141],"experiments":[142],"LeNet-5":[144],"MNIST,":[146],"ResNet-18,":[147],"VGG-19":[148],"Cifar-10,":[150],"Seq2Seq":[152],"Multi30k.":[154],"Results":[155],"show":[156],"that":[157],"GSSP":[158],"accelerates":[159],"by":[162],"46%$\\sim$120%,":[163],"less":[165],"than":[166],"1":[167],"percent":[168],"accuracy":[169],"drop.":[170],"And":[171],"GDTopK":[172],"up":[175],"part":[177],"of":[178],"lost":[180],"accuracy.":[181]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":4},{"year":2021,"cited_by_count":1}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-10-10T00:00:00"}