{"id":"https://openalex.org/W2908133769","doi":"https://doi.org/10.1162/neco_a_01248","title":"A Continuous-Time Analysis of Distributed Stochastic Gradient","display_name":"A Continuous-Time Analysis of Distributed Stochastic Gradient","publication_year":2019,"publication_date":"2019-11-09","ids":{"openalex":"https://openalex.org/W2908133769","doi":"https://doi.org/10.1162/neco_a_01248","mag":"2908133769","pmid":"https://pubmed.ncbi.nlm.nih.gov/31703177"},"language":"en","primary_location":{"id":"doi:10.1162/neco_a_01248","is_oa":true,"landing_page_url":"https://doi.org/10.1162/neco_a_01248","pdf_url":"https://direct.mit.edu/neco/article-pdf/32/1/36/1864518/neco_a_01248.pdf","source":{"id":"https://openalex.org/S207023548","display_name":"Neural Computation","issn_l":"0899-7667","issn":["0899-7667","1530-888X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310315718","host_organization_name":"The MIT Press","host_organization_lineage":["https://openalex.org/P4310315718"],"host_organization_lineage_names":["The MIT Press"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural Computation","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://direct.mit.edu/neco/article-pdf/32/1/36/1864518/neco_a_01248.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Nicholas M. Boffi","orcid":null},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Nicholas M. Boffi","raw_affiliation_strings":["John A. Paulson School of Engineering and Applied Sciences, Harvard University, Cambridge, MA 02138, U.S.A"],"affiliations":[{"raw_affiliation_string":"John A. Paulson School of Engineering and Applied Sciences, Harvard University, Cambridge, MA 02138, U.S.A","institution_ids":["https://openalex.org/I136199984"]}]},{"author_position":"last","author":{"id":null,"display_name":"Jean-Jacques E. Slotine","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jean-Jacques E. Slotine","raw_affiliation_strings":["Nonlinear Systems Laboratory, MIT, Cambridge, MA 02139, U.S.A"],"affiliations":[{"raw_affiliation_string":"Nonlinear Systems Laboratory, MIT, Cambridge, MA 02139, U.S.A","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I136199984"],"apc_list":null,"apc_paid":null,"fwci":0.7414,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.73044664,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":"32","issue":"1","first_page":"36","last_page":"96"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10249","display_name":"Distributed Control Multi-Agent Systems","score":0.2759000062942505,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10249","display_name":"Distributed Control Multi-Agent Systems","score":0.2759000062942505,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11347","display_name":"Neural Networks Stability and Synchronization","score":0.11159999668598175,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.09700000286102295,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/smoothing","display_name":"Smoothing","score":0.6128000020980835},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.5569999814033508},{"id":"https://openalex.org/keywords/convex-function","display_name":"Convex function","score":0.47850000858306885},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.4763000011444092},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.46880000829696655},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.4596000015735626},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.3982999920845032},{"id":"https://openalex.org/keywords/dynamical-systems-theory","display_name":"Dynamical systems theory","score":0.3806999921798706},{"id":"https://openalex.org/keywords/variance-reduction","display_name":"Variance reduction","score":0.3806000053882599},{"id":"https://openalex.org/keywords/reduction","display_name":"Reduction (mathematics)","score":0.36629998683929443}],"concepts":[{"id":"https://openalex.org/C3770464","wikidata":"https://www.wikidata.org/wiki/Q775963","display_name":"Smoothing","level":2,"score":0.6128000020980835},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.5569999814033508},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.48840001225471497},{"id":"https://openalex.org/C145446738","wikidata":"https://www.wikidata.org/wiki/Q319913","display_name":"Convex function","level":3,"score":0.47850000858306885},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.4763000011444092},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.46880000829696655},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.4596000015735626},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.44530001282691956},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4429999887943268},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.3982999920845032},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.38339999318122864},{"id":"https://openalex.org/C79379906","wikidata":"https://www.wikidata.org/wiki/Q3174497","display_name":"Dynamical systems theory","level":2,"score":0.3806999921798706},{"id":"https://openalex.org/C62644790","wikidata":"https://www.wikidata.org/wiki/Q3454689","display_name":"Variance reduction","level":3,"score":0.3806000053882599},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.36629998683929443},{"id":"https://openalex.org/C130120984","wikidata":"https://www.wikidata.org/wiki/Q2835898","display_name":"Distributed algorithm","level":2,"score":0.36160001158714294},{"id":"https://openalex.org/C157972887","wikidata":"https://www.wikidata.org/wiki/Q463359","display_name":"Convex optimization","level":3,"score":0.34470000863075256},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3416000008583069},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.335099995136261},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.32280001044273376},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.3165000081062317},{"id":"https://openalex.org/C194387892","wikidata":"https://www.wikidata.org/wiki/Q1747770","display_name":"Stochastic optimization","level":2,"score":0.31529998779296875},{"id":"https://openalex.org/C55479107","wikidata":"https://www.wikidata.org/wiki/Q97663916","display_name":"Stochastic approximation","level":3,"score":0.30820000171661377},{"id":"https://openalex.org/C131584629","wikidata":"https://www.wikidata.org/wiki/Q4308705","display_name":"Coupling (piping)","level":2,"score":0.30630001425743103},{"id":"https://openalex.org/C115680565","wikidata":"https://www.wikidata.org/wiki/Q5977448","display_name":"Gradient method","level":2,"score":0.3057999908924103},{"id":"https://openalex.org/C111097370","wikidata":"https://www.wikidata.org/wiki/Q10969923","display_name":"Synchronization networks","level":4,"score":0.2935999929904938},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.28369998931884766},{"id":"https://openalex.org/C8272713","wikidata":"https://www.wikidata.org/wiki/Q176737","display_name":"Stochastic process","level":2,"score":0.2808000147342682},{"id":"https://openalex.org/C163294075","wikidata":"https://www.wikidata.org/wiki/Q581861","display_name":"Noise reduction","level":2,"score":0.27399998903274536},{"id":"https://openalex.org/C91873725","wikidata":"https://www.wikidata.org/wiki/Q3445816","display_name":"Function approximation","level":3,"score":0.266400009393692},{"id":"https://openalex.org/C4199805","wikidata":"https://www.wikidata.org/wiki/Q2725903","display_name":"Gaussian noise","level":2,"score":0.2653999924659729},{"id":"https://openalex.org/C16210155","wikidata":"https://www.wikidata.org/wiki/Q5154140","display_name":"Communication noise","level":2,"score":0.26339998841285706},{"id":"https://openalex.org/C75438885","wikidata":"https://www.wikidata.org/wiki/Q3403615","display_name":"Large deviations theory","level":2,"score":0.25540000200271606},{"id":"https://openalex.org/C86582703","wikidata":"https://www.wikidata.org/wiki/Q7617824","display_name":"Stochastic neural network","level":4,"score":0.25380000472068787},{"id":"https://openalex.org/C112680207","wikidata":"https://www.wikidata.org/wiki/Q714886","display_name":"Regular polygon","level":2,"score":0.25270000100135803}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1162/neco_a_01248","is_oa":true,"landing_page_url":"https://doi.org/10.1162/neco_a_01248","pdf_url":"https://direct.mit.edu/neco/article-pdf/32/1/36/1864518/neco_a_01248.pdf","source":{"id":"https://openalex.org/S207023548","display_name":"Neural Computation","issn_l":"0899-7667","issn":["0899-7667","1530-888X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310315718","host_organization_name":"The MIT Press","host_organization_lineage":["https://openalex.org/P4310315718"],"host_organization_lineage_names":["The MIT Press"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural Computation","raw_type":"journal-article"},{"id":"pmid:31703177","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/31703177","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural computation","raw_type":null},{"id":"pmh:oai:arXiv.org:1812.10995","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1812.10995","pdf_url":"https://arxiv.org/pdf/1812.10995","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1162/neco_a_01248","is_oa":true,"landing_page_url":"https://doi.org/10.1162/neco_a_01248","pdf_url":"https://direct.mit.edu/neco/article-pdf/32/1/36/1864518/neco_a_01248.pdf","source":{"id":"https://openalex.org/S207023548","display_name":"Neural Computation","issn_l":"0899-7667","issn":["0899-7667","1530-888X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310315718","host_organization_name":"The MIT Press","host_organization_lineage":["https://openalex.org/P4310315718"],"host_organization_lineage_names":["The MIT Press"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural Computation","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2908133769.pdf","grobid_xml":"https://content.openalex.org/works/W2908133769.grobid-xml"},"referenced_works_count":30,"referenced_works":["https://openalex.org/W114517082","https://openalex.org/W1561789346","https://openalex.org/W1571545895","https://openalex.org/W2025492232","https://openalex.org/W2040038761","https://openalex.org/W2050118726","https://openalex.org/W2070718355","https://openalex.org/W2086161653","https://openalex.org/W2096504654","https://openalex.org/W2105767123","https://openalex.org/W2110538250","https://openalex.org/W2121599014","https://openalex.org/W2124541940","https://openalex.org/W2126922170","https://openalex.org/W2131869707","https://openalex.org/W2134889984","https://openalex.org/W2140380216","https://openalex.org/W2165861322","https://openalex.org/W2261689926","https://openalex.org/W2276533724","https://openalex.org/W2790013445","https://openalex.org/W2919115771","https://openalex.org/W2963156201","https://openalex.org/W2963177640","https://openalex.org/W2963430672","https://openalex.org/W2963480765","https://openalex.org/W2963862297","https://openalex.org/W4229706427","https://openalex.org/W4231021340","https://openalex.org/W4292363360"],"related_works":[],"abstract_inverted_index":{"We":[0,97,144,167],"analyze":[1],"the":[2,41,44,48,67,70,75,111,115,119,122,132,138,163,181,198],"effect":[3],"of":[4,18,43,69,114,118,159,192],"synchronization":[5,23,37],"on":[6,81,110,129,175,180],"distributed":[7,50],"stochastic":[8,76],"gradient":[9,77],"algorithms.":[10],"By":[11],"exploiting":[12],"an":[13,126],"analogy":[14],"with":[15,30,63,135,172,215],"dynamical":[16],"models":[17],"biological":[19],"quorum":[20,130],"sensing,":[21,131],"where":[22,185],"between":[24],"agents":[25,51,120],"is":[26,59],"induced":[27],"through":[28],"communication":[29],"a":[31,64,99,108,188],"common":[32],"signal,":[33],"we":[34,85,186],"quantify":[35],"how":[36],"can":[38,89],"significantly":[39],"reduce":[40],"magnitude":[42],"noise":[45,57,92],"felt":[46],"by":[47,74,106],"individual":[49],"and":[52,94,137,161],"their":[53],"spatial":[54,116],"mean.":[55],"This":[56,201],"reduction":[58,65],"in":[60,66,206],"turn":[61],"associated":[62],"smoothing":[68],"loss":[71],"function":[72],"imposed":[73],"approximation.":[78],"Through":[79],"simulations":[80],"model":[82],"nonconvex":[83],"objectives,":[84],"demonstrate":[86],"that":[87,150,212],"coupling":[88],"stabilize":[90],"higher":[91],"levels":[93],"improve":[95],"convergence.":[96],"provide":[98],"convergence":[100],"analysis":[101,171],"for":[102,125,209],"strongly":[103],"convex":[104],"functions":[105],"deriving":[107],"bound":[109],"expected":[112],"deviation":[113],"mean":[117],"from":[121],"global":[123],"minimizer":[124],"algorithm":[127,134],"based":[128],"same":[133],"momentum,":[136],"elastic":[139],"averaging":[140],"SGD":[141],"(EASGD)":[142],"algorithm.":[143],"discuss":[145],"extensions":[146],"to":[147,154,197],"new":[148],"algorithms":[149,208],"allow":[151],"each":[152],"agent":[153],"broadcast":[155],"its":[156],"current":[157],"measure":[158],"success":[160],"shape":[162],"collective":[164],"computation":[165],"accordingly.":[166],"supplement":[168],"our":[169],"theoretical":[170],"numerical":[173],"experiments":[174],"convolutional":[176],"neural":[177],"networks":[178],"trained":[179],"CIFAR-10":[182],"data":[183],"set,":[184],"note":[187],"surprising":[189],"regularizing":[190],"property":[191],"EASGD":[193],"even":[194],"when":[195],"applied":[196],"non-distributed":[199],"case.":[200],"observation":[202],"suggests":[203],"alternative":[204],"second-order":[205],"time":[207],"nondistributed":[210],"optimization":[211],"are":[213],"competitive":[214],"momentum":[216],"methods.":[217]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2019-01-11T00:00:00"}