{"id":"https://openalex.org/W3022621540","doi":"https://doi.org/10.1109/msp.2020.2975212","title":"Asymptotic Network Independence in Distributed Stochastic Optimization for Machine Learning: Examining Distributed and Centralized Stochastic Gradient Descent","display_name":"Asymptotic Network Independence in Distributed Stochastic Optimization for Machine Learning: Examining Distributed and Centralized Stochastic Gradient Descent","publication_year":2020,"publication_date":"2020-05-01","ids":{"openalex":"https://openalex.org/W3022621540","doi":"https://doi.org/10.1109/msp.2020.2975212","mag":"3022621540","pmid":"https://pubmed.ncbi.nlm.nih.gov/33746471"},"language":"en","primary_location":{"id":"doi:10.1109/msp.2020.2975212","is_oa":false,"landing_page_url":"https://doi.org/10.1109/msp.2020.2975212","pdf_url":null,"source":{"id":"https://openalex.org/S120977877","display_name":"IEEE Signal Processing Magazine","issn_l":"1053-5888","issn":["1053-5888","1558-0792"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Signal Processing Magazine","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/7977622","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032734653","display_name":"Shi Pu","orcid":"https://orcid.org/0000-0002-5813-527X"},"institutions":[{"id":"https://openalex.org/I111088046","display_name":"Boston University","ror":"https://ror.org/05qwgg493","country_code":"US","type":"education","lineage":["https://openalex.org/I111088046"]},{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Shi Pu","raw_affiliation_strings":["Institute for Data and Decision Analytics, The Chinese University of Hong Kong, Shenzhen, China and Shenzhen Research Institute of Big Data. The research was conducted when the author was with Division of Systems Engineering, Boston University, Boston, MA","Postdoctoral Associate, Boston University, Massachusetts, USA"],"raw_orcid":"https://orcid.org/0000-0002-5813-527X","affiliations":[{"raw_affiliation_string":"Institute for Data and Decision Analytics, The Chinese University of Hong Kong, Shenzhen, China and Shenzhen Research Institute of Big Data. The research was conducted when the author was with Division of Systems Engineering, Boston University, Boston, MA","institution_ids":["https://openalex.org/I4210116924"]},{"raw_affiliation_string":"Postdoctoral Associate, Boston University, Massachusetts, USA","institution_ids":["https://openalex.org/I111088046"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005863931","display_name":"Alex Olshevsky","orcid":"https://orcid.org/0000-0002-5852-9789"},"institutions":[{"id":"https://openalex.org/I111088046","display_name":"Boston University","ror":"https://ror.org/05qwgg493","country_code":"US","type":"education","lineage":["https://openalex.org/I111088046"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alex Olshevsky","raw_affiliation_strings":["Department of Electrical and Computer Engineering and Division of Systems Engineering, Boston University, Boston, MA","Department of Electrical and Computer Engineering, Boston University, Massachusetts, USA"],"raw_orcid":"https://orcid.org/0000-0002-5852-9789","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering and Division of Systems Engineering, Boston University, Boston, MA","institution_ids":["https://openalex.org/I111088046"]},{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Boston University, Massachusetts, USA","institution_ids":["https://openalex.org/I111088046"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075696701","display_name":"Ioannis Ch. Paschalidis","orcid":"https://orcid.org/0000-0002-3343-2913"},"institutions":[{"id":"https://openalex.org/I111088046","display_name":"Boston University","ror":"https://ror.org/05qwgg493","country_code":"US","type":"education","lineage":["https://openalex.org/I111088046"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ioannis Ch. Paschalidis","raw_affiliation_strings":["Department of Electrical and Computer Engineering and Division of Systems Engineering, Boston University, Boston, MA","Information and Systems Engineering, Boston University, Massachusetts, USA"],"raw_orcid":"https://orcid.org/0000-0002-3343-2913","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering and Division of Systems Engineering, Boston University, Boston, MA","institution_ids":["https://openalex.org/I111088046"]},{"raw_affiliation_string":"Information and Systems Engineering, Boston University, Massachusetts, USA","institution_ids":["https://openalex.org/I111088046"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.7915,"has_fulltext":false,"cited_by_count":56,"citation_normalized_percentile":{"value":0.94458068,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"37","issue":"3","first_page":"114","last_page":"122"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10249","display_name":"Distributed Control Multi-Agent Systems","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10500","display_name":"Sparse and Compressive Sensing Techniques","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.785412073135376},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.681842029094696},{"id":"https://openalex.org/keywords/independence","display_name":"Independence (probability theory)","score":0.6484807133674622},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.6383981704711914},{"id":"https://openalex.org/keywords/stochastic-optimization","display_name":"Stochastic optimization","score":0.4627869427204132},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.4359331727027893},{"id":"https://openalex.org/keywords/descent","display_name":"Descent (aeronautics)","score":0.4356904923915863},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3798424005508423},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.34965980052948},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.28457099199295044},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1975877583026886},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08052930235862732}],"concepts":[{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.785412073135376},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.681842029094696},{"id":"https://openalex.org/C35651441","wikidata":"https://www.wikidata.org/wiki/Q625303","display_name":"Independence (probability theory)","level":2,"score":0.6484807133674622},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.6383981704711914},{"id":"https://openalex.org/C194387892","wikidata":"https://www.wikidata.org/wiki/Q1747770","display_name":"Stochastic optimization","level":2,"score":0.4627869427204132},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4359331727027893},{"id":"https://openalex.org/C2776637919","wikidata":"https://www.wikidata.org/wiki/Q624380","display_name":"Descent (aeronautics)","level":2,"score":0.4356904923915863},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3798424005508423},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.34965980052948},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.28457099199295044},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1975877583026886},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08052930235862732},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/msp.2020.2975212","is_oa":false,"landing_page_url":"https://doi.org/10.1109/msp.2020.2975212","pdf_url":null,"source":{"id":"https://openalex.org/S120977877","display_name":"IEEE Signal Processing Magazine","issn_l":"1053-5888","issn":["1053-5888","1558-0792"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Signal Processing Magazine","raw_type":"journal-article"},{"id":"pmid:33746471","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/33746471","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE signal processing magazine","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:7977622","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/7977622","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Signal Process Mag","raw_type":"Text"}],"best_oa_location":{"id":"pmh:oai:pubmedcentral.nih.gov:7977622","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/7977622","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Signal Process Mag","raw_type":"Text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2376368022","display_name":null,"funder_award_id":"1R01GM135930","funder_id":"https://openalex.org/F4320337432","funder_display_name":"Center for Scientific Review"},{"id":"https://openalex.org/G2782532677","display_name":null,"funder_award_id":"DMS-1664644","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3024551116","display_name":null,"funder_award_id":"N00014-19-1-2571","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G3889586898","display_name":null,"funder_award_id":"ECCS-1933027","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6946221972","display_name":null,"funder_award_id":"IIS-1914792","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7025932738","display_name":null,"funder_award_id":"CNS- 1645681","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"},{"id":"https://openalex.org/F4320337432","display_name":"Center for Scientific Review","ror":"https://ror.org/04r5s4b52"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":46,"referenced_works":["https://openalex.org/W1992208280","https://openalex.org/W2027481851","https://openalex.org/W2029080014","https://openalex.org/W2044212084","https://openalex.org/W2056182476","https://openalex.org/W2081543628","https://openalex.org/W2082904296","https://openalex.org/W2112269233","https://openalex.org/W2140655807","https://openalex.org/W2586683640","https://openalex.org/W2609549560","https://openalex.org/W2617960902","https://openalex.org/W2763533254","https://openalex.org/W2771477296","https://openalex.org/W2783522756","https://openalex.org/W2805185133","https://openalex.org/W2893687496","https://openalex.org/W2899734758","https://openalex.org/W2902410171","https://openalex.org/W2911327153","https://openalex.org/W2962757173","https://openalex.org/W2962771678","https://openalex.org/W2962917900","https://openalex.org/W2963023064","https://openalex.org/W2963076621","https://openalex.org/W2963156126","https://openalex.org/W2963228337","https://openalex.org/W2963515790","https://openalex.org/W2963855576","https://openalex.org/W2964263318","https://openalex.org/W2975682251","https://openalex.org/W2989361410","https://openalex.org/W3009274618","https://openalex.org/W3026733282","https://openalex.org/W3100513551","https://openalex.org/W4288612157","https://openalex.org/W4299802343","https://openalex.org/W6676963778","https://openalex.org/W6738250615","https://openalex.org/W6755506522","https://openalex.org/W6756200373","https://openalex.org/W6756976665","https://openalex.org/W6758557334","https://openalex.org/W6768174580","https://openalex.org/W6770379143","https://openalex.org/W6774136274"],"related_works":["https://openalex.org/W4206903459","https://openalex.org/W2754816816","https://openalex.org/W4366280654","https://openalex.org/W3160167280","https://openalex.org/W4231621013","https://openalex.org/W4362706668","https://openalex.org/W2015288657","https://openalex.org/W3008318776","https://openalex.org/W1977633006","https://openalex.org/W4286899070"],"abstract_inverted_index":{"We":[0,72],"provide":[1],"a":[2,16,39,44,56,60,87],"discussion":[3],"of":[4,46,82,95],"several":[5],"recent":[6],"results":[7],"which,":[8],"in":[9,18],"certain":[10],"scenarios,":[11],"are":[12],"able":[13],"to":[14,51,59],"overcome":[15],"barrier":[17],"distributed":[19,40,96],"stochastic":[20,97],"optimization":[21],"for":[22,91],"machine":[23],"learning":[24],"(ML).":[25],"Our":[26],"focus":[27],"is":[28,36],"the":[29,52,64,69,80,93],"so-called":[30],"asymptotic":[31],"network":[32,45],"independence":[33],"property,":[34],"which":[35],"achieved":[37],"whenever":[38],"method":[41,62],"executed":[42],"over":[43],"n":[47],"nodes":[48],"asymptotically":[49],"converges":[50],"optimal":[53],"solution":[54],"at":[55],"comparable":[57],"rate":[58],"centralized":[61,102],"with":[63,101],"same":[65],"computational":[66],"power":[67],"as":[68],"entire":[70],"network.":[71],"explain":[73],"this":[74],"property":[75],"through":[76],"an":[77],"example":[78],"involving":[79],"training":[81],"ML":[83],"models":[84],"and":[85],"sketch":[86],"short":[88],"mathematical":[89],"analysis":[90],"comparing":[92],"performance":[94],"gradient":[98],"descent":[99],"(DSGD)":[100],"SGD.":[103]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":17},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":10},{"year":2019,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2020-05-13T00:00:00"}