{"id":"https://openalex.org/W3153822571","doi":"https://doi.org/10.1109/access.2021.3071579","title":"Bridging the Gap Between Memory and Communication Efficiency on Distributed Deep Learning Systems","display_name":"Bridging the Gap Between Memory and Communication Efficiency on Distributed Deep Learning Systems","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3153822571","doi":"https://doi.org/10.1109/access.2021.3071579","mag":"3153822571"},"language":"en","primary_location":{"id":"doi:10.1109/access.2021.3071579","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3071579","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09398682.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09398682.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Shaofeng Zhao","orcid":null},"institutions":[{"id":"https://openalex.org/I4210138186","display_name":"Wuhan National Laboratory for Optoelectronics","ror":"https://ror.org/03c9ncn37","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210138186"]},{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]},{"id":"https://openalex.org/I7726996","display_name":"Henan University of Economic and Law","ror":"https://ror.org/000jtc944","country_code":"CN","type":"education","lineage":["https://openalex.org/I7726996"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shaofeng Zhao","raw_affiliation_strings":["Cyberspace Administration Center, Henan University of Economics and Law, Zhengzhou, China","Key Laboratory of Information Storage System, Engineering Research Center of Data Storage Systems and Technology, Huazhong University of Science and Technology, Wuhan, China","Wuhan National Laboratory for Optoelectronics, Huazhong University of Science and Technology, Wuhan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cyberspace Administration Center, Henan University of Economics and Law, Zhengzhou, China","institution_ids":["https://openalex.org/I7726996"]},{"raw_affiliation_string":"Key Laboratory of Information Storage System, Engineering Research Center of Data Storage Systems and Technology, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]},{"raw_affiliation_string":"Wuhan National Laboratory for Optoelectronics, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I4210138186","https://openalex.org/I47720641"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051934365","display_name":"Bo Liu","orcid":"https://orcid.org/0000-0002-8908-0777"},"institutions":[{"id":"https://openalex.org/I7726996","display_name":"Henan University of Economic and Law","ror":"https://ror.org/000jtc944","country_code":"CN","type":"education","lineage":["https://openalex.org/I7726996"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bo Liu","raw_affiliation_strings":["Cyberspace Administration Center, Henan University of Economics and Law, Zhengzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-8908-0777","affiliations":[{"raw_affiliation_string":"Cyberspace Administration Center, Henan University of Economics and Law, Zhengzhou, China","institution_ids":["https://openalex.org/I7726996"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100321449","display_name":"Fang Wang","orcid":"https://orcid.org/0000-0002-9355-0292"},"institutions":[{"id":"https://openalex.org/I4210138186","display_name":"Wuhan National Laboratory for Optoelectronics","ror":"https://ror.org/03c9ncn37","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210138186"]},{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fang Wang","raw_affiliation_strings":["Key Laboratory of Information Storage System, Engineering Research Center of Data Storage Systems and Technology, Huazhong University of Science and Technology, Wuhan, China","Wuhan National Laboratory for Optoelectronics, Huazhong University of Science and Technology, Wuhan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Key Laboratory of Information Storage System, Engineering Research Center of Data Storage Systems and Technology, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]},{"raw_affiliation_string":"Wuhan National Laboratory for Optoelectronics, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I4210138186","https://openalex.org/I47720641"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057421680","display_name":"Dan Feng","orcid":"https://orcid.org/0000-0002-4674-6006"},"institutions":[{"id":"https://openalex.org/I4210138186","display_name":"Wuhan National Laboratory for Optoelectronics","ror":"https://ror.org/03c9ncn37","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210138186"]},{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dan Feng","raw_affiliation_strings":["Key Laboratory of Information Storage System, Engineering Research Center of Data Storage Systems and Technology, Huazhong University of Science and Technology, Wuhan, China","Wuhan National Laboratory for Optoelectronics, Huazhong University of Science and Technology, Wuhan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Key Laboratory of Information Storage System, Engineering Research Center of Data Storage Systems and Technology, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]},{"raw_affiliation_string":"Wuhan National Laboratory for Optoelectronics, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I4210138186","https://openalex.org/I47720641"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.03474138,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"9","issue":null,"first_page":"57075","last_page":"57088"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12702","display_name":"Brain Tumor Detection and Classification","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/2808","display_name":"Neurology"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.879607081413269},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.6650335788726807},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.6104747653007507},{"id":"https://openalex.org/keywords/bandwidth","display_name":"Bandwidth (computing)","score":0.4581330120563507},{"id":"https://openalex.org/keywords/memory-bandwidth","display_name":"Memory bandwidth","score":0.4483790993690491},{"id":"https://openalex.org/keywords/bridging","display_name":"Bridging (networking)","score":0.4403679668903351},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.43112921714782715},{"id":"https://openalex.org/keywords/distributed-memory","display_name":"Distributed memory","score":0.4100250005722046},{"id":"https://openalex.org/keywords/shared-memory","display_name":"Shared memory","score":0.39367157220840454},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.3518781065940857},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2617366909980774},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.2113802433013916},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.15559789538383484}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.879607081413269},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.6650335788726807},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.6104747653007507},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.4581330120563507},{"id":"https://openalex.org/C188045654","wikidata":"https://www.wikidata.org/wiki/Q17148339","display_name":"Memory bandwidth","level":2,"score":0.4483790993690491},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.4403679668903351},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.43112921714782715},{"id":"https://openalex.org/C91481028","wikidata":"https://www.wikidata.org/wiki/Q1054686","display_name":"Distributed memory","level":3,"score":0.4100250005722046},{"id":"https://openalex.org/C133875982","wikidata":"https://www.wikidata.org/wiki/Q764810","display_name":"Shared memory","level":2,"score":0.39367157220840454},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3518781065940857},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2617366909980774},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.2113802433013916},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.15559789538383484}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2021.3071579","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3071579","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09398682.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:122da6eca8484cccb868d254fe7074c6","is_oa":false,"landing_page_url":"https://doaj.org/article/122da6eca8484cccb868d254fe7074c6","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 9, Pp 57075-57088 (2021)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2021.3071579","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3071579","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09398682.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1756779261","display_name":"\u57fa\u4e8e\u5e94\u7528\u957f\u65f6\u7a7a\u57df\u8fd0\u884c\u7279\u5f81\u7684\u56fe\u6570\u636e\u5b58\u50a8\u7ec4\u7ec7\u53ca\u7cfb\u7edf\u4f18\u5316\u7814\u7a76","funder_award_id":"61772216","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2413230818","display_name":null,"funder_award_id":"61821003","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6147031788","display_name":null,"funder_award_id":"2018YFB1003305","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G7584325168","display_name":null,"funder_award_id":"61832020","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3153822571.pdf","grobid_xml":"https://content.openalex.org/works/W3153822571.grobid-xml"},"referenced_works_count":58,"referenced_works":["https://openalex.org/W1614298861","https://openalex.org/W1686810756","https://openalex.org/W1960027552","https://openalex.org/W2108598243","https://openalex.org/W2153579005","https://openalex.org/W2163605009","https://openalex.org/W2186615578","https://openalex.org/W2194775991","https://openalex.org/W2262962221","https://openalex.org/W2293634267","https://openalex.org/W2339765813","https://openalex.org/W2402144811","https://openalex.org/W2407022425","https://openalex.org/W2489529491","https://openalex.org/W2606891064","https://openalex.org/W2617766261","https://openalex.org/W2769644379","https://openalex.org/W2774000609","https://openalex.org/W2786680212","https://openalex.org/W2803113791","https://openalex.org/W2891993230","https://openalex.org/W2896457183","https://openalex.org/W2950577311","https://openalex.org/W2953384591","https://openalex.org/W2962739339","https://openalex.org/W2962835968","https://openalex.org/W2963341956","https://openalex.org/W2963403868","https://openalex.org/W2963803379","https://openalex.org/W2964004663","https://openalex.org/W2964163156","https://openalex.org/W2964174152","https://openalex.org/W2979719709","https://openalex.org/W3012514909","https://openalex.org/W3101036738","https://openalex.org/W3103894541","https://openalex.org/W3111155334","https://openalex.org/W3118608800","https://openalex.org/W4294170691","https://openalex.org/W4301361180","https://openalex.org/W4385245566","https://openalex.org/W6636510571","https://openalex.org/W6637373629","https://openalex.org/W6640876409","https://openalex.org/W6682691769","https://openalex.org/W6684191040","https://openalex.org/W6686509673","https://openalex.org/W6696934422","https://openalex.org/W6713134421","https://openalex.org/W6714239094","https://openalex.org/W6738460352","https://openalex.org/W6739901393","https://openalex.org/W6746200960","https://openalex.org/W6746839373","https://openalex.org/W6751528251","https://openalex.org/W6751646691","https://openalex.org/W6755207826","https://openalex.org/W6787972765"],"related_works":["https://openalex.org/W2026512611","https://openalex.org/W1985165680","https://openalex.org/W4245497162","https://openalex.org/W2353146130","https://openalex.org/W1990817968","https://openalex.org/W2150064838","https://openalex.org/W2135766592","https://openalex.org/W2185094550","https://openalex.org/W2088823210","https://openalex.org/W2398725611"],"abstract_inverted_index":{"Large-scale":[0],"distributed":[1,13,110,195,228,289],"deep":[2,46,173],"learning":[3],"is":[4,32,78,132,142,152,267,295],"of":[5,39,53,69,74,93,98,138,178,182,227,258,280],"great":[6],"importance":[7],"in":[8,96,188,194],"various":[9],"applications.":[10],"For":[11],"data-parallel":[12],"training":[14,197,229,256],"systems,":[15],"limited":[16],"hardware":[17],"resources":[18,41],"(e.g.,":[19],"GPU":[20,266],"memory":[21,57,101,120,129,140,159,212,247,310],"and":[22,30,59,102,105,134,144,161,175,218,224,253,291],"interconnection":[23],"bandwidth)":[24],"often":[25],"become":[26],"a":[27,71,107,147,237,264],"performance":[28],"bottleneck,":[29],"it":[31],"necessary":[33],"to":[34,65,117,154,190,243,251],"consider":[35,90],"the":[36,67,91,119,127,135,139,156,172,186,204,211,215,221,225,245,254,259,277,281,292],"full":[37],"utilization":[38,223],"multiple":[40,191],"simultaneously,":[42],"especially":[43],"for":[44,165],"extreme-scale":[45],"neural":[47,260],"networks.":[48],"Although":[49],"two":[50,76,180],"different":[51],"types":[52,181],"strategies,":[54,183],"based":[55],"on":[56,214,263],"management":[58,160],"sparse":[60,111,125,162],"communication,":[61,126],"have":[62],"been":[63],"proposed":[64,153,205],"reduce":[66,244],"usage":[68,248],"resources,":[70,104],"na\u00efve":[72],"combination":[73],"these":[75,179],"optimizations":[77],"impractical,":[79],"since":[80],"they":[81],"cannot":[82],"successfully":[83],"coexist":[84],"with":[85,232],"each":[86],"other.":[87],"We":[88],"therefore":[89],"idea":[92],"collaborative":[94,176,206],"optimization":[95,141,164,193,207,313],"terms":[97],"both":[99,220],"system":[100,246],"bandwidth":[103],"propose":[106],"layer-centric":[108],"memory-efficient":[109],"communication":[112,163,312],"mechanism":[113,151],"called":[114],"LaySA.":[115],"Firstly,":[116],"tackle":[118],"ballooning":[121],"issue":[122],"caused":[123],"by":[124,249,269,283,297],"existing":[128],"reuse":[130],"strategy":[131],"refined,":[133],"data":[136],"object":[137],"augmented":[143],"redefined.":[145],"Secondly,":[146],"mirror":[148],"weight":[149,166],"update":[150],"address":[155],"contradiction":[157],"between":[158],"gradients.":[167],"Our":[168,199],"scheme,":[169],"which":[170],"involves":[171],"integration":[174],"execution":[177],"can":[184,208,241,274],"fill":[185],"gap":[187],"relation":[189],"resource":[192,222],"GPU-based":[196],"systems.":[198,230],"experimental":[200],"results":[201],"show":[202],"that":[203,302,308],"significantly":[209],"alleviate":[210],"pressure":[213],"computing":[216],"nodes,":[217],"improve":[219],"throughput":[226,294],"Compared":[231],"baseline":[233],"systems":[234,307],"using":[235],"only":[236],"single":[238,265],"strategy,":[239],"LaySA":[240,273],"help":[242],"up":[250,276],"80.5%,":[252],"overall":[255,293],"time":[257],"network":[261],"models":[262],"reduced":[268],"about":[270],"12.25%.":[271],"Furthermore,":[272],"scale":[275],"batch":[278],"size":[279],"datasets":[282],"an":[284],"extremely":[285],"large":[286],"factor":[287],"during":[288],"training,":[290],"increased":[296],"more":[298],"than":[299],"150%,":[300],"meaning":[301],"our":[303],"approach":[304],"outperforms":[305],"current":[306],"use":[309],"or":[311],"mechanisms":[314],"alone.":[315]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}