{"id":"https://openalex.org/W3048338200","doi":"https://doi.org/10.1145/3404397.3404432","title":"Developing a Loss Prediction-based Asynchronous Stochastic Gradient Descent Algorithm for Distributed Training of Deep Neural Networks","display_name":"Developing a Loss Prediction-based Asynchronous Stochastic Gradient Descent Algorithm for Distributed Training of Deep Neural Networks","publication_year":2020,"publication_date":"2020-08-09","ids":{"openalex":"https://openalex.org/W3048338200","doi":"https://doi.org/10.1145/3404397.3404432","mag":"3048338200"},"language":"en","primary_location":{"id":"doi:10.1145/3404397.3404432","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404397.3404432","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"49th International Conference on Parallel Processing - ICPP","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://wrap.warwick.ac.uk/140145/1/WRAP-Developing-loss-prediction-based-asynchronous-stochastic-Li-2020.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100680719","display_name":"Junyu Li","orcid":"https://orcid.org/0000-0002-9972-3561"},"institutions":[{"id":"https://openalex.org/I39555362","display_name":"University of Warwick","ror":"https://ror.org/01a77tt86","country_code":"GB","type":"education","lineage":["https://openalex.org/I39555362"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Junyu Li","raw_affiliation_strings":["University of Warwick"],"affiliations":[{"raw_affiliation_string":"University of Warwick","institution_ids":["https://openalex.org/I39555362"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064596191","display_name":"Ligang He","orcid":"https://orcid.org/0000-0002-5671-0576"},"institutions":[{"id":"https://openalex.org/I39555362","display_name":"University of Warwick","ror":"https://ror.org/01a77tt86","country_code":"GB","type":"education","lineage":["https://openalex.org/I39555362"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ligang He","raw_affiliation_strings":["University of Warwick"],"affiliations":[{"raw_affiliation_string":"University of Warwick","institution_ids":["https://openalex.org/I39555362"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113345812","display_name":"Shenyuan Ren","orcid":null},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Shenyuan Ren","raw_affiliation_strings":["University of Oxford"],"affiliations":[{"raw_affiliation_string":"University of Oxford","institution_ids":["https://openalex.org/I40120149"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101724957","display_name":"Rui Mao","orcid":"https://orcid.org/0000-0002-3645-5520"},"institutions":[{"id":"https://openalex.org/I180726961","display_name":"Shenzhen University","ror":"https://ror.org/01vy4gh70","country_code":"CN","type":"education","lineage":["https://openalex.org/I180726961"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rui Mao","raw_affiliation_strings":["Shenzhen University"],"affiliations":[{"raw_affiliation_string":"Shenzhen University","institution_ids":["https://openalex.org/I180726961"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5100680719"],"corresponding_institution_ids":["https://openalex.org/I39555362"],"apc_list":null,"apc_paid":null,"fwci":0.6628,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":{"value":0.7588791,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8112688064575195},{"id":"https://openalex.org/keywords/asynchronous-communication","display_name":"Asynchronous communication","score":0.749021589756012},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6770356297492981},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.6501927375793457},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.6483937501907349},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.536949634552002},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5217921733856201},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5135438442230225},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.4818076491355896},{"id":"https://openalex.org/keywords/node","display_name":"Node (physics)","score":0.46190008521080017},{"id":"https://openalex.org/keywords/hessian-matrix","display_name":"Hessian matrix","score":0.434866726398468},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4130588471889496},{"id":"https://openalex.org/keywords/compensation","display_name":"Compensation (psychology)","score":0.4112360179424286},{"id":"https://openalex.org/keywords/real-time-computing","display_name":"Real-time computing","score":0.38210925459861755},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.380935937166214},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.12762975692749023},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.07019689679145813}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8112688064575195},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.749021589756012},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6770356297492981},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.6501927375793457},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.6483937501907349},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.536949634552002},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5217921733856201},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5135438442230225},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4818076491355896},{"id":"https://openalex.org/C62611344","wikidata":"https://www.wikidata.org/wiki/Q1062658","display_name":"Node (physics)","level":2,"score":0.46190008521080017},{"id":"https://openalex.org/C203616005","wikidata":"https://www.wikidata.org/wiki/Q620495","display_name":"Hessian matrix","level":2,"score":0.434866726398468},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4130588471889496},{"id":"https://openalex.org/C2780023022","wikidata":"https://www.wikidata.org/wiki/Q1338171","display_name":"Compensation (psychology)","level":2,"score":0.4112360179424286},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.38210925459861755},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.380935937166214},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.12762975692749023},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07019689679145813},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C66938386","wikidata":"https://www.wikidata.org/wiki/Q633538","display_name":"Structural engineering","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C11171543","wikidata":"https://www.wikidata.org/wiki/Q41630","display_name":"Psychoanalysis","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3404397.3404432","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404397.3404432","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"49th International Conference on Parallel Processing - ICPP","raw_type":"proceedings-article"},{"id":"pmh:oai:wrap.warwick.ac.uk:140145","is_oa":true,"landing_page_url":null,"pdf_url":"http://wrap.warwick.ac.uk/140145/1/WRAP-Developing-loss-prediction-based-asynchronous-stochastic-Li-2020.pdf","source":{"id":"https://openalex.org/S4306400665","display_name":"Warwick Research Archive Portal (University of Warwick)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I39555362","host_organization_name":"University of Warwick","host_organization_lineage":["https://openalex.org/I39555362"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Item"}],"best_oa_location":{"id":"pmh:oai:wrap.warwick.ac.uk:140145","is_oa":true,"landing_page_url":null,"pdf_url":"http://wrap.warwick.ac.uk/140145/1/WRAP-Developing-loss-prediction-based-asynchronous-stochastic-Li-2020.pdf","source":{"id":"https://openalex.org/S4306400665","display_name":"Warwick Research Archive Portal (University of Warwick)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I39555362","host_organization_name":"University of Warwick","host_organization_lineage":["https://openalex.org/I39555362"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Item"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth","score":0.4300000071525574}],"awards":[{"id":"https://openalex.org/G7282611729","display_name":null,"funder_award_id":"EP/R007195/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3048338200.pdf","grobid_xml":"https://content.openalex.org/works/W3048338200.grobid-xml"},"referenced_works_count":24,"referenced_works":["https://openalex.org/W2060393849","https://openalex.org/W2127941149","https://openalex.org/W2168231600","https://openalex.org/W2188647300","https://openalex.org/W2194775991","https://openalex.org/W2271840356","https://openalex.org/W2405883473","https://openalex.org/W2484096406","https://openalex.org/W2513289469","https://openalex.org/W2541884796","https://openalex.org/W2626580042","https://openalex.org/W2769644379","https://openalex.org/W2774769449","https://openalex.org/W2802446276","https://openalex.org/W2911002035","https://openalex.org/W2912213068","https://openalex.org/W2916979304","https://openalex.org/W2949117887","https://openalex.org/W2951781666","https://openalex.org/W2963318081","https://openalex.org/W2963327228","https://openalex.org/W2963433607","https://openalex.org/W2964004663","https://openalex.org/W2995191368"],"related_works":["https://openalex.org/W2355987247","https://openalex.org/W3143650729","https://openalex.org/W3177326532","https://openalex.org/W4297883503","https://openalex.org/W2042173174","https://openalex.org/W4206903459","https://openalex.org/W4283028824","https://openalex.org/W2754816816","https://openalex.org/W4366280654","https://openalex.org/W3160167280"],"abstract_inverted_index":{"Training":[0],"Deep":[1],"Neural":[2],"Network":[3],"is":[4,15,103,168],"a":[5,33,38,62,73,113,196],"computation-intensive":[6],"and":[7,88,106,149,174],"time-consuming":[8],"task.":[9],"Asynchronous":[10],"Stochastic":[11],"Gradient":[12],"Descent":[13],"(ASGD)":[14],"an":[16,82],"effective":[17],"solution":[18],"to":[19,29,118,153],"accelerate":[20],"the":[21,27,42,53,59,66,77,86,89,101,121,130,135,146,151,156,171,191],"training":[22],"process":[23],"since":[24],"it":[25],"enables":[26],"network":[28],"be":[30],"trained":[31,194],"in":[32,145],"distributed":[34,94],"fashion,":[35],"but":[36],"with":[37,72,81,195],"main":[39],"issue":[40],"of":[41,55,65,199],"delayed":[43],"gradient":[44,97],"update.":[45],"A":[46],"recent":[47],"notable":[48],"work":[49],"called":[50,116],"DC-ASGD":[51,69],"improves":[52,185],"performance":[54,78],"ASGD":[56],"by":[57,100],"compensating":[58],"delay":[60,84,98,132],"using":[61],"cheap":[63],"approximation":[64],"Hessian":[67],"matrix.":[68],"works":[70],"well":[71],"short":[74],"delay;":[75],"however,":[76],"drops":[79],"considerably":[80],"increasing":[83],"between":[85],"workers":[87],"server.":[90],"In":[91,108],"real-life":[92],"large-scale":[93],"training,":[95],"such":[96],"experienced":[99],"worker":[102],"usually":[104],"high":[105],"volatile.":[107],"this":[109],"paper,":[110],"we":[111],"propose":[112],"novel":[114],"algorithm":[115,167],"LC-ASGD":[117,139,183],"compensate":[119,154],"for":[120,134,155],"delay,":[122,157],"basing":[123,158],"on":[124,159,170],"Loss":[125],"Prediction.":[126],"It":[127],"effectively":[128],"extends":[129],"tolerable":[131],"duration":[133],"compensation":[136],"mechanism.":[137],"Specifically,":[138],"utilizes":[140],"additional":[141],"models":[142],"that":[143,181],"reside":[144],"parameter":[147],"server":[148],"predict":[150],"loss":[152],"historical":[160],"losses":[161],"collected":[162],"from":[163],"each":[164],"worker.":[165],"The":[166,177],"evaluated":[169],"popular":[172],"networks":[173,192],"benchmark":[175],"datasets.":[176],"experimental":[178],"results":[179],"show":[180],"our":[182],"significantly":[184],"over":[186],"existing":[187],"methods,":[188],"especially":[189],"when":[190],"are":[193],"large":[197],"number":[198],"workers.":[200]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}