{"id":"https://openalex.org/W2221532093","doi":null,"title":"Model Accuracy and Runtime Tradeoff in Distributed Deep Learning","display_name":"Model Accuracy and Runtime Tradeoff in Distributed Deep Learning","publication_year":2015,"publication_date":"2015-09-14","ids":{"openalex":"https://openalex.org/W2221532093","mag":"2221532093"},"language":"en","primary_location":{"id":"mag:2221532093","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/1509.04210","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://export.arxiv.org/pdf/1509.04210","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5110305071","display_name":"Suyog Gupta","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Suyog Gupta","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Wei Zhang","orcid":"https://orcid.org/0000-0002-2937-1732","observed_orcids":["https://orcid.org/0000-0002-2937-1732"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei Zhang","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-2937-1732","affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5034216883","display_name":"Josh Milthorpe","orcid":"https://orcid.org/0000-0002-3588-9896","observed_orcids":["https://orcid.org/0000-0002-3588-9896"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Josh Milthorpe","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-3588-9896","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":16,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9473000168800354,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9473000168800354,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.02019999921321869,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.00989999994635582,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/distributed-deep-learning","display_name":"distributed deep learning","score":0.9900000095367432},{"id":"https://openalex.org/keywords/parameter-server","display_name":"parameter server","score":0.7350000143051147},{"id":"https://openalex.org/keywords/asynchronous-stochastic-gradient-descent","display_name":"asynchronous stochastic gradient descent","score":0.9399999976158142},{"id":"https://openalex.org/keywords/synchronization-protocols","display_name":"synchronization protocols","score":0.753000020980835},{"id":"https://openalex.org/keywords/learning-rate-modulation","display_name":"learning rate modulation","score":0.9470000267028809},{"id":"https://openalex.org/keywords/mini-batch-size","display_name":"mini-batch size","score":0.9229999780654907},{"id":"https://openalex.org/keywords/model-accuracy","display_name":"model accuracy","score":0.5960000157356262},{"id":"https://openalex.org/keywords/runtime-performance","display_name":"runtime performance","score":0.8629999756813049},{"id":"https://openalex.org/keywords/large-scale-neural-network-training","display_name":"large-scale neural network training","score":0.7099999785423279}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8416991233825684},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.8175579309463501},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.7166986465454102},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.6917188167572021},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.6385955810546875},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5153052806854248},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5023858547210693},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4713327884674072},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.442754864692688},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4321565330028534},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4060204029083252},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.11130255460739136},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C204787440","wikidata":"https://www.wikidata.org/wiki/Q188504","display_name":"Alternative medicine","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:2221532093","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/1509.04210","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:2221532093","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/1509.04210","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W6908809","https://openalex.org/W104184427","https://openalex.org/W778657980","https://openalex.org/W1442374986","https://openalex.org/W1533861849","https://openalex.org/W1677182931","https://openalex.org/W1806891645","https://openalex.org/W1814095264","https://openalex.org/W1833917188","https://openalex.org/W1845277745","https://openalex.org/W2000200144","https://openalex.org/W2041517243","https://openalex.org/W2091590079","https://openalex.org/W2117539524","https://openalex.org/W2132737349","https://openalex.org/W2146502635","https://openalex.org/W2162390675","https://openalex.org/W2163605009","https://openalex.org/W2168231600","https://openalex.org/W2265775419","https://openalex.org/W2919115771","https://openalex.org/W2949117887","https://openalex.org/W3118608800","https://openalex.org/W3137220996","https://openalex.org/W7098642242"],"related_works":["https://openalex.org/W2568564101","https://openalex.org/W2898323977","https://openalex.org/W2982664135","https://openalex.org/W4283694394","https://openalex.org/W4205519540","https://openalex.org/W2527027103","https://openalex.org/W2788814033","https://openalex.org/W7203724885","https://openalex.org/W3206218057","https://openalex.org/W4415792991"],"abstract_inverted_index":{"This":[0],"paper":[1],"presents":[2],"Rudra,":[3],"a":[4,53,68,92],"parameter":[5],"server":[6],"based":[7],"distributed":[8,96],"computing":[9],"framework":[10],"tuned":[11],"for":[12,95],"training":[13,97],"large-scale":[14],"deep":[15],"neural":[16,99],"networks.":[17],"Using":[18],"variants":[19],"of":[20,31,43,63,98],"the":[21,29,61,76,101,115,119],"asynchronous":[22],"stochastic":[23],"gradient":[24,35],"descent":[25],"algorithm":[26],"we":[27],"study":[28],"impact":[30],"synchronization":[32,70],"protocol,":[33],"stale":[34,64],"updates,":[36],"minibatch":[37],"size,":[38],"learning":[39,55],"rates,":[40],"and":[41,48,66,83,132],"number":[42],"learners":[44,111],"on":[45],"runtime":[46,81],"performance":[47,82],"model":[49,86,120],"accuracy.":[50,87,121],"We":[51,122],"introduce":[52],"new":[54,69],"rate":[56],"modulation":[57],"strategy":[58],"to":[59,114,117],"counter":[60],"effect":[62],"gradients":[65],"propose":[67],"protocol":[71],"that":[72],"can":[73],"effectively":[74],"bound":[75],"staleness":[77],"in":[78],"gradients,":[79],"improve":[80],"achieve":[84],"good":[85],"Our":[88],"empirical":[89],"investigation":[90],"reveals":[91],"principled":[93],"approach":[94,125],"networks:":[100],"mini-batch":[102],"size":[103],"per":[104],"learner":[105],"should":[106],"be":[107],"reduced":[108],"as":[109],"more":[110],"are":[112],"added":[113],"system":[116],"preserve":[118],"validate":[123],"this":[124],"using":[126],"commonly-used":[127],"image":[128],"classification":[129],"benchmarks:":[130],"CIFAR10":[131],"ImageNet.":[133]},"counts_by_year":[{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":6},{"year":2015,"cited_by_count":2}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}