{"id":"https://openalex.org/W2900182564","doi":"https://doi.org/10.1609/aaai.v33i01.33015693","title":"Parallel Restarted SGD with Faster Convergence and Less Communication: Demystifying Why Model Averaging Works for Deep Learning","display_name":"Parallel Restarted SGD with Faster Convergence and Less Communication: Demystifying Why Model Averaging Works for Deep Learning","publication_year":2019,"publication_date":"2019-07-17","ids":{"openalex":"https://openalex.org/W2900182564","doi":"https://doi.org/10.1609/aaai.v33i01.33015693","mag":"2900182564"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v33i01.33015693","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v33i01.33015693","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1609/aaai.v33i01.33015693","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055267677","display_name":"Hao Yu","orcid":"https://orcid.org/0000-0002-7261-5448"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hao Yu","raw_affiliation_strings":["Alibaba Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002854172","display_name":"Sen Yang","orcid":"https://orcid.org/0000-0003-3222-2268"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sen Yang","raw_affiliation_strings":["Alibaba Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103861122","display_name":"Shenghuo Zhu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shenghuo Zhu","raw_affiliation_strings":["Alibaba Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210095624"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":512,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"33","issue":"01","first_page":"5693","last_page":"5700"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9884999990463257,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.758465051651001},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7046704292297363},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.6879357099533081},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.6582551002502441},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.5564252138137817},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.5471266508102417},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5206781625747681},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.510225772857666},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.47176674008369446},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.4401637315750122},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.42798054218292236},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.41455674171447754},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3968675434589386},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3940317630767822},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.34120747447013855}],"concepts":[{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.758465051651001},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7046704292297363},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.6879357099533081},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.6582551002502441},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.5564252138137817},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.5471266508102417},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5206781625747681},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.510225772857666},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.47176674008369446},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.4401637315750122},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.42798054218292236},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.41455674171447754},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3968675434589386},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3940317630767822},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.34120747447013855},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1609/aaai.v33i01.33015693","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v33i01.33015693","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/4514","is_oa":true,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/4514","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/4514/4392","source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v33i01.33015693","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v33i01.33015693","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.44999998807907104,"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":39,"referenced_works":["https://openalex.org/W855165587","https://openalex.org/W1779452081","https://openalex.org/W1843828865","https://openalex.org/W2054959047","https://openalex.org/W2108475251","https://openalex.org/W2127941149","https://openalex.org/W2130062883","https://openalex.org/W2138302120","https://openalex.org/W2166706236","https://openalex.org/W2168231600","https://openalex.org/W2405578611","https://openalex.org/W2405883473","https://openalex.org/W2407022425","https://openalex.org/W2460815216","https://openalex.org/W2563343794","https://openalex.org/W2617766261","https://openalex.org/W2617960902","https://openalex.org/W2622263826","https://openalex.org/W2702150584","https://openalex.org/W2765170276","https://openalex.org/W2769644379","https://openalex.org/W2886404071","https://openalex.org/W2888206291","https://openalex.org/W2962696932","https://openalex.org/W2962863496","https://openalex.org/W2962952793","https://openalex.org/W2963179579","https://openalex.org/W2963470657","https://openalex.org/W2963804082","https://openalex.org/W2963889845","https://openalex.org/W3101036738","https://openalex.org/W3102816259","https://openalex.org/W3212066104","https://openalex.org/W4230874317","https://openalex.org/W4297797495","https://openalex.org/W4301239768","https://openalex.org/W4318619660","https://openalex.org/W6638803421","https://openalex.org/W6684249991"],"related_works":["https://openalex.org/W2058965144","https://openalex.org/W2164382479","https://openalex.org/W98480971","https://openalex.org/W2499279132","https://openalex.org/W271331623","https://openalex.org/W2966297898","https://openalex.org/W2955229517","https://openalex.org/W1974690493","https://openalex.org/W4312723892","https://openalex.org/W2783065147"],"abstract_inverted_index":{"In":[0],"distributed":[1,124],"training":[2,17,70,125,171],"of":[3,68,77,126,148,169],"deep":[4,127],"neural":[5,128],"networks,":[6],"parallel":[7,60,116,142,215],"minibatch":[8],"SGD":[9,53,62,81,217],"is":[10,92,118,151,179],"widely":[11],"used":[12,122],"to":[13,27,41,74],"speed":[14],"up":[15],"the":[16,43,56,69,75,96,145,170,176],"process":[18],"by":[19,95],"using":[20,51],"multiple":[21,25],"workers.":[22],"It":[23],"uses":[24],"workers":[26,104],"sample":[28],"local":[29,49],"stochastic":[30],"gradients":[31,36],"in":[32,37,90,187],"parallel,":[33],"aggregates":[34],"all":[35],"a":[38,52,65,83,166,185,191,200],"single":[39,84],"server":[40],"obtain":[42],"average,":[44],"and":[45,137,202],"updates":[46],"each":[47],"worker\u2019s":[48],"model":[50,149,161,208],"update":[54],"with":[55,80,141,218],"averaged":[57],"gradient.":[58],"Ideally,":[59],"mini-batch":[61,143,216],"can":[63,163,210],"achieve":[64,165],"linear":[66,88],"speed-up":[67,168],"time":[71,172],"(with":[72],"respect":[73],"number":[76],"workers)":[78],"compared":[79],"over":[82,115],"worker.":[85],"However,":[86,182],"such":[87,190],"scalability":[89],"practice":[91,121],"significantly":[93,152,219],"limited":[94],"growing":[97],"demand":[98],"for":[99,123],"gradient":[100],"communication":[101,146,221],"as":[102,173,175,212,214],"more":[103],"are":[105],"involved.":[106],"Model":[107],"averaging,":[108],"which":[109],"periodically":[110],"averages":[111],"individual":[112],"models":[113],"trained":[114],"workers,":[117],"another":[119],"common":[120],"networks":[129],"since":[130],"(Zinkevich":[131],"et":[132],"al.":[133],"2010)":[134],"(McDonald,":[135],"Hall,":[136],"Mann":[138],"2010).":[139],"Compared":[140],"SGD,":[144],"overhead":[147],"averaging":[150,162,177,209],"reduced.":[153],"Impressively,":[154],"tremendous":[155],"experimental":[156],"works":[157,194],"have":[158],"verified":[159],"that":[160],"still":[164],"good":[167],"long":[174],"interval":[178],"carefully":[180],"controlled.":[181],"it":[183],"remains":[184],"mystery":[186],"theory":[188],"why":[189,207],"simple":[192],"heuristic":[193],"so":[195],"well.":[196],"This":[197],"paper":[198],"provides":[199],"thorough":[201],"rigorous":[203],"theoretical":[204],"study":[205],"on":[206],"work":[211],"well":[213],"less":[220],"overhead.":[222]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":50},{"year":2024,"cited_by_count":72},{"year":2023,"cited_by_count":94},{"year":2022,"cited_by_count":81},{"year":2021,"cited_by_count":110},{"year":2020,"cited_by_count":75},{"year":2019,"cited_by_count":23},{"year":2018,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}