{"id":"https://openalex.org/W4294037873","doi":"https://doi.org/10.48550/arxiv.1708.02188","title":"PowerAI DDL","display_name":"PowerAI DDL","publication_year":2017,"publication_date":"2017-08-07","ids":{"openalex":"https://openalex.org/W4294037873","doi":"https://doi.org/10.48550/arxiv.1708.02188","mag":"3037716762"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:1708.02188","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1708.02188","pdf_url":"https://arxiv.org/pdf/1708.02188","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1708.02188","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5115076654","display_name":"Minsik Cho","orcid":"https://orcid.org/0000-0003-0481-2682","observed_orcids":["https://orcid.org/0000-0003-0481-2682"]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cho, Minsik","raw_affiliation_strings":["IBM Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000286196","display_name":"Ulrich Finkler","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Finkler, Ulrich","raw_affiliation_strings":["IBM Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050468445","display_name":"Sameer Kumar","orcid":"https://orcid.org/0000-0003-4592-9502","observed_orcids":["https://orcid.org/0000-0003-4592-9502"]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kumar, Sameer","raw_affiliation_strings":["IBM Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071497092","display_name":"David S. Kung","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kung, David","raw_affiliation_strings":["IBM Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044459787","display_name":"Vaibhav Saxena","orcid":"https://orcid.org/0000-0002-4533-785X","observed_orcids":["https://orcid.org/0000-0002-4533-785X"]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saxena, Vaibhav","raw_affiliation_strings":["IBM Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5104093269","display_name":"Dheeraj Sreedhar","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sreedhar, Dheeraj","raw_affiliation_strings":["IBM Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research","institution_ids":["https://openalex.org/I1341412227"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1341412227"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":31,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9560999870300293,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9560999870300293,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.012000000104308128,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.00839999970048666,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/distributed-deep-learning","display_name":"distributed deep learning","score":0.9919999837875366},{"id":"https://openalex.org/keywords/hardware-software-co-design","display_name":"hardware-software co-design","score":0.9110000133514404},{"id":"https://openalex.org/keywords/gpu-training","display_name":"GPU training","score":0.652999997138977},{"id":"https://openalex.org/keywords/resnet-101","display_name":"ResNet-101","score":0.6729999780654907},{"id":"https://openalex.org/keywords/resnet-50","display_name":"ResNet50","score":0.8379999995231628},{"id":"https://openalex.org/keywords/training-time-reduction","display_name":"training time reduction","score":0.4519999921321869},{"id":"https://openalex.org/keywords/communication-library","display_name":"communication library","score":0.3370000123977661}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8244666457176208},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.626567006111145},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5975537300109863},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.5722587704658508},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.549323558807373},{"id":"https://openalex.org/C70388272","wikidata":"https://www.wikidata.org/wiki/Q5968558","display_name":"IBM","level":2,"score":0.5381653308868408},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5333659052848816},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5266551971435547},{"id":"https://openalex.org/C2944601119","wikidata":"https://www.wikidata.org/wiki/Q43744058","display_name":"Residual neural network","level":3,"score":0.4504179358482361},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.4476901888847351},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.4404246211051941},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.4268912672996521},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.322601854801178},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.3137568533420563},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.13534796237945557},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C171250308","wikidata":"https://www.wikidata.org/wiki/Q11468","display_name":"Nanotechnology","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:1708.02188","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1708.02188","pdf_url":"https://arxiv.org/pdf/1708.02188","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3037716762","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1708.02188","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1708.02188","doi":"https://doi.org/10.48550/arxiv.1708.02188","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1708.02188","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1708.02188","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1708.02188","pdf_url":"https://arxiv.org/pdf/1708.02188","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4294037873.pdf","grobid_xml":"https://content.openalex.org/works/W4294037873.grobid-xml"},"referenced_works_count":6,"referenced_works":["https://openalex.org/W2155893237","https://openalex.org/W2168231600","https://openalex.org/W2194775991","https://openalex.org/W2402144811","https://openalex.org/W2949650786","https://openalex.org/W7098642242"],"related_works":["https://openalex.org/W2398934890","https://openalex.org/W2204075824","https://openalex.org/W2626721309","https://openalex.org/W3175449831","https://openalex.org/W2963281229","https://openalex.org/W2898323977","https://openalex.org/W4312644250","https://openalex.org/W2915943718","https://openalex.org/W2170135819","https://openalex.org/W2994144272"],"abstract_inverted_index":{"As":[0],"deep":[1,22],"neural":[2,23],"networks":[3],"become":[4],"more":[5],"complex":[6],"and":[7,92,94,121,151,182,186],"input":[8],"datasets":[9],"grow":[10],"larger,":[11],"it":[12,42],"can":[13,68],"take":[14],"days":[15],"or":[16],"even":[17],"weeks":[18,52],"to":[19,25,46,53,73,96,141,166],"train":[20,124],"a":[21,34,38,60,81,87,97,108,178],"network":[24],"the":[26,44,48],"desired":[27],"accuracy.":[28,148],"Therefore,":[29],"distributed":[30,63],"Deep":[31,64],"Learning":[32,65],"at":[33],"massive":[35],"scale":[36],"is":[37,80,105],"critical":[39],"capability,":[40],"since":[41],"offers":[43,189],"potential":[45],"reduce":[47],"training":[49,206],"time":[50],"from":[51],"hours.":[54],"In":[55],"this":[56],"paper,":[57],"we":[58,176],"present":[59],"software-hardware":[61],"co-optimized":[62],"system":[66,100,188],"that":[67,85],"achieve":[69],"near-linear":[70],"scaling":[71],"up":[72],"hundreds":[74],"of":[75,99,144,200,205],"GPUs.":[76],"The":[77,102],"core":[78],"algorithm":[79,104],"multi-ring":[82],"communication":[83,103,180,191],"pattern":[84],"provides":[86],"good":[88],"tradeoff":[89],"between":[90],"latency":[91],"bandwidth":[93],"adapts":[95],"variety":[98],"configurations.":[101],"implemented":[106],"as":[107],"library":[109,114],"for":[110,162,193,210],"easy":[111],"use.":[112],"This":[113],"has":[115],"been":[116],"integrated":[117],"into":[118],"Tensorflow,":[119],"Caffe,":[120],"Torch.":[122],"We":[123],"Resnet-101":[125],"on":[126,172,207],"Imagenet":[127,163],"22K":[128],"with":[129],"64":[130,217],"IBM":[131,218],"Power8":[132,219],"S822LC":[133,220],"servers":[134,221],"(256":[135,222],"GPUs)":[136],"in":[137,213],"about":[138],"7":[139],"hours":[140],"an":[142],"accuracy":[143,161],"33.8":[145],"%":[146,159],"validation":[147,160],"Microsoft's":[149],"ADAM":[150],"Google's":[152],"DistBelief":[153],"results":[154],"did":[155],"not":[156],"reach":[157],"30":[158],"22K.":[164],"Compared":[165],"Facebook":[167],"AI":[168],"Research's":[169],"recent":[170],"paper":[171],"256":[173],"GPU":[174],"training,":[175],"use":[177],"different":[179],"algorithm,":[181],"our":[183],"combined":[184],"software":[185],"hardware":[187],"better":[190],"overhead":[192],"Resnet-50.":[194],"A":[195],"PowerAI":[196],"DDL":[197],"enabled":[198],"version":[199],"Torch":[201],"completed":[202],"90":[203],"epochs":[204],"Resnet":[208],"50":[209,214],"1K":[211],"classes":[212],"minutes":[215],"using":[216],"GPUs).":[223]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":9},{"year":2019,"cited_by_count":4},{"year":2018,"cited_by_count":12},{"year":2017,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2022-09-01T00:00:00"}