{"id":"https://openalex.org/W2765985261","doi":"https://doi.org/10.1145/3146347.3146353","title":"Towards Scalable Parallel Training of Deep Neural Networks","display_name":"Towards Scalable Parallel Training of Deep Neural Networks","publication_year":2017,"publication_date":"2017-10-31","ids":{"openalex":"https://openalex.org/W2765985261","doi":"https://doi.org/10.1145/3146347.3146353","mag":"2765985261"},"language":"en","primary_location":{"id":"doi:10.1145/3146347.3146353","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3146347.3146353","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3146353&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Machine Learning on HPC Environments","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"http://dl.acm.org/ft_gateway.cfm?id=3146353&type=pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5017294111","display_name":"Sam Ad\u00e9 Jacobs","orcid":"https://orcid.org/0000-0003-3425-5602"},"institutions":[{"id":"https://openalex.org/I1282311441","display_name":"Lawrence Livermore National Laboratory","ror":"https://ror.org/041nk4h53","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282311441","https://openalex.org/I1330989302","https://openalex.org/I198811213","https://openalex.org/I4210138311"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sam Ad\u00e9 Jacobs","raw_affiliation_strings":["Lawrence Livermore National Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Lawrence Livermore National Laboratory","institution_ids":["https://openalex.org/I1282311441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066306669","display_name":"Nikoli Dryden","orcid":null},"institutions":[{"id":"https://openalex.org/I1282311441","display_name":"Lawrence Livermore National Laboratory","ror":"https://ror.org/041nk4h53","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282311441","https://openalex.org/I1330989302","https://openalex.org/I198811213","https://openalex.org/I4210138311"]},{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nikoli Dryden","raw_affiliation_strings":["Department of Computer Science, University of Illinois at Urbana-Champaign, Lawrence Livermore National Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Illinois at Urbana-Champaign, Lawrence Livermore National Laboratory","institution_ids":["https://openalex.org/I1282311441","https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046256174","display_name":"Roger Pearce","orcid":"https://orcid.org/0009-0000-6449-6568"},"institutions":[{"id":"https://openalex.org/I1282311441","display_name":"Lawrence Livermore National Laboratory","ror":"https://ror.org/041nk4h53","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282311441","https://openalex.org/I1330989302","https://openalex.org/I198811213","https://openalex.org/I4210138311"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Roger Pearce","raw_affiliation_strings":["Lawrence Livermore National Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Lawrence Livermore National Laboratory","institution_ids":["https://openalex.org/I1282311441"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5110927536","display_name":"Brian Van Essen","orcid":null},"institutions":[{"id":"https://openalex.org/I1282311441","display_name":"Lawrence Livermore National Laboratory","ror":"https://ror.org/041nk4h53","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282311441","https://openalex.org/I1330989302","https://openalex.org/I198811213","https://openalex.org/I4210138311"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Brian Van Essen","raw_affiliation_strings":["Lawrence Livermore National Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Lawrence Livermore National Laboratory","institution_ids":["https://openalex.org/I1282311441"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":14,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8522621393203735},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.7218460440635681},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.7192058563232422},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6691526174545288},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.6515138149261475},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.645226240158081},{"id":"https://openalex.org/keywords/ibm","display_name":"IBM","score":0.6123707890510559},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.542198657989502},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5298884510993958},{"id":"https://openalex.org/keywords/tournament","display_name":"Tournament","score":0.4900287389755249},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.487648606300354},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.4874556064605713},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.42171138525009155},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.38621124625205994},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.15764716267585754},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.10218778252601624}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8522621393203735},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.7218460440635681},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.7192058563232422},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6691526174545288},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6515138149261475},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.645226240158081},{"id":"https://openalex.org/C70388272","wikidata":"https://www.wikidata.org/wiki/Q5968558","display_name":"IBM","level":2,"score":0.6123707890510559},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.542198657989502},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5298884510993958},{"id":"https://openalex.org/C136975688","wikidata":"https://www.wikidata.org/wiki/Q1320634","display_name":"Tournament","level":2,"score":0.4900287389755249},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.487648606300354},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.4874556064605713},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.42171138525009155},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.38621124625205994},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.15764716267585754},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.10218778252601624},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C171250308","wikidata":"https://www.wikidata.org/wiki/Q11468","display_name":"Nanotechnology","level":1,"score":0.0},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3146347.3146353","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3146347.3146353","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3146353&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Machine Learning on HPC Environments","raw_type":"proceedings-article"},{"id":"pmh:oai:osti.gov:1410093","is_oa":true,"landing_page_url":"https://www.osti.gov/biblio/1410093","pdf_url":null,"source":{"id":"https://openalex.org/S4306402487","display_name":"OSTI OAI (U.S. Department of Energy Office of Scientific and Technical Information)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I139351228","host_organization_name":"Office of Scientific and Technical Information","host_organization_lineage":["https://openalex.org/I139351228"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"doi:10.1145/3146347.3146353","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3146347.3146353","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3146353&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Machine Learning on HPC Environments","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4343739756","display_name":null,"funder_award_id":"Contract DE-AC52-07NA27344","funder_id":"https://openalex.org/F4320338286","funder_display_name":"Lawrence Livermore National Laboratory"},{"id":"https://openalex.org/G4844777956","display_name":null,"funder_award_id":"DE-AC52-07NA27344","funder_id":"https://openalex.org/F4320337547","funder_display_name":"Laboratory Directed Research and Development"},{"id":"https://openalex.org/G5522089832","display_name":null,"funder_award_id":"DE-AC52-07NA27344","funder_id":"https://openalex.org/F4320338286","funder_display_name":"Lawrence Livermore National Laboratory"},{"id":"https://openalex.org/G5816449448","display_name":null,"funder_award_id":"DE-AC52-07NA27344","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G69709960","display_name":null,"funder_award_id":"Contract DE-AC52-07NA27344","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G8374966015","display_name":null,"funder_award_id":"AC52-07NA27344","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G991281326","display_name":null,"funder_award_id":"AC52-07NA27344","funder_id":"https://openalex.org/F4320338286","funder_display_name":"Lawrence Livermore National Laboratory"}],"funders":[{"id":"https://openalex.org/F4320306084","display_name":"U.S. Department of Energy","ror":"https://ror.org/01bj3aw27"},{"id":"https://openalex.org/F4320337547","display_name":"Laboratory Directed Research and Development","ror":"https://ror.org/01e41cf67"},{"id":"https://openalex.org/F4320338286","display_name":"Lawrence Livermore National Laboratory","ror":"https://ror.org/041nk4h53"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2765985261.pdf","grobid_xml":"https://content.openalex.org/works/W2765985261.grobid-xml"},"referenced_works_count":33,"referenced_works":["https://openalex.org/W22040386","https://openalex.org/W203719604","https://openalex.org/W1845277745","https://openalex.org/W1922655562","https://openalex.org/W2016053056","https://openalex.org/W2097117768","https://openalex.org/W2118858186","https://openalex.org/W2120432001","https://openalex.org/W2130325614","https://openalex.org/W2155893237","https://openalex.org/W2160815625","https://openalex.org/W2163605009","https://openalex.org/W2168231600","https://openalex.org/W2170135819","https://openalex.org/W2172734211","https://openalex.org/W2184045248","https://openalex.org/W2250904038","https://openalex.org/W2253807446","https://openalex.org/W2294929133","https://openalex.org/W2295582178","https://openalex.org/W2308045930","https://openalex.org/W2336650964","https://openalex.org/W2400777838","https://openalex.org/W2405578611","https://openalex.org/W2407022425","https://openalex.org/W2523060838","https://openalex.org/W2557283755","https://openalex.org/W2563343794","https://openalex.org/W2616180702","https://openalex.org/W2622263826","https://openalex.org/W2951781666","https://openalex.org/W2962911728","https://openalex.org/W4230874317"],"related_works":["https://openalex.org/W3123786980","https://openalex.org/W2994965880","https://openalex.org/W4287997709","https://openalex.org/W3083287393","https://openalex.org/W3094108612","https://openalex.org/W3125314910","https://openalex.org/W2982074041","https://openalex.org/W2130950591","https://openalex.org/W2041858633","https://openalex.org/W3122959053"],"abstract_inverted_index":{"We":[0,119],"propose":[1],"a":[2,42,75,110],"new":[3,95],"framework":[4,26,150],"for":[5],"parallelizing":[6],"deep":[7,107,116],"neural":[8,108],"network":[9],"training":[10,22,88,92,106],"that":[11,17],"maximize":[12],"the":[13,21,90,148,168],"amount":[14,101],"of":[15,44,54,102,147],"data":[16,36],"is":[18,85],"ingested":[19],"by":[20],"algorithm.":[23],"Our":[24],"proposed":[25,122],"called":[27],"Livermore":[28,130],"Tournament":[29],"Fast":[30],"Batch":[31],"Learning":[32],"(LTFB)":[33],"targets":[34],"large-scale":[35],"problems.":[37],"The":[38,82],"LTFB":[39,149],"approach":[40,96],"creates":[41],"set":[43],"Deep":[45],"Neural":[46],"Network":[47],"(DNN)":[48],"models":[49,56],"and":[50,58,72,99,159],"trains":[51],"each":[52,62],"instance":[53],"these":[55],"independently":[57],"in":[59,105,113],"parallel.":[60],"Periodically":[61],"model":[63,66,84],"selects":[64],"another":[65],"to":[67,167],"pair":[68],"with,":[69],"exchanges":[70],"models,":[71],"then":[73],"run":[74],"local":[76,91],"tournament":[77,80],"against":[78],"held-out":[79],"datasets.":[81,93],"winning":[83],"will":[86],"continue":[87],"on":[89,124,151],"This":[94],"maximizes":[97],"computation":[98],"minimizes":[100],"synchronization":[103],"required":[104],"network,":[109],"major":[111],"bottleneck":[112],"existing":[114],"synchronous":[115],"learning":[117],"algorithms.":[118],"evaluate":[120],"our":[121],"algorithm":[123],"two":[125,152],"HPC":[126],"machines":[127],"at":[128],"Lawrence":[129],"National":[131],"Laboratory":[132],"including":[133],"an":[134],"early":[135],"access":[136],"IBM":[137],"Power8+":[138],"with":[139],"NVIDIA":[140],"Tesla":[141],"P100":[142],"GPUs":[143],"machine.":[144],"Experimental":[145],"evaluations":[146],"popular":[153],"image":[154],"classification":[155],"benchmark:":[156],"CIFAR10":[157],"[18]":[158],"ImageNet":[160],"[19],":[161],"show":[162],"significant":[163],"speed":[164],"up":[165],"compared":[166],"sequential":[169],"baseline.":[170]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":2},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}