{"id":"https://openalex.org/W3120480394","doi":"https://doi.org/10.1109/tpds.2020.3048836","title":"Why Dataset Properties Bound the Scalability of Parallel Machine Learning Training Algorithms","display_name":"Why Dataset Properties Bound the Scalability of Parallel Machine Learning Training Algorithms","publication_year":2021,"publication_date":"2021-01-07","ids":{"openalex":"https://openalex.org/W3120480394","doi":"https://doi.org/10.1109/tpds.2020.3048836","mag":"3120480394"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2020.3048836","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2020.3048836","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010944123","display_name":"Daning Cheng","orcid":"https://orcid.org/0000-0002-9217-6980"},"institutions":[{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Daning Cheng","raw_affiliation_strings":["SKL, Institute of Computing Technology, Chinese Academy of Science, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9217-6980","affiliations":[{"raw_affiliation_string":"SKL, Institute of Computing Technology, Chinese Academy of Science, Beijing, China","institution_ids":["https://openalex.org/I4210090176"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088172355","display_name":"Shigang Li","orcid":"https://orcid.org/0000-0003-0022-7865"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Shigang Li","raw_affiliation_strings":["Department of Computer Science, ETH, Zurich, ZH, Switzerland"],"raw_orcid":"https://orcid.org/0000-0003-0022-7865","affiliations":[{"raw_affiliation_string":"Department of Computer Science, ETH, Zurich, ZH, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Hanping Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210134214","display_name":"Beijing Research Institute of Uranium Geology","ror":"https://ror.org/046qx3a23","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210134214"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hanping Zhang","raw_affiliation_strings":["Algorithm Department, Beijing Wisdom Uranium Technology Co., Ltd., Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Algorithm Department, Beijing Wisdom Uranium Technology Co., Ltd., Beijing, China","institution_ids":["https://openalex.org/I4210134214"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100532486","display_name":"Fen Xia","orcid":null},"institutions":[{"id":"https://openalex.org/I4210134214","display_name":"Beijing Research Institute of Uranium Geology","ror":"https://ror.org/046qx3a23","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210134214"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fen Xia","raw_affiliation_strings":["Algorithm Department, Beijing Wisdom Uranium Technology Co., Ltd., Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Algorithm Department, Beijing Wisdom Uranium Technology Co., Ltd., Beijing, China","institution_ids":["https://openalex.org/I4210134214"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001666028","display_name":"Yunquan Zhang","orcid":"https://orcid.org/0000-0002-2618-5088"},"institutions":[{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yunquan Zhang","raw_affiliation_strings":["SKL of Computer Architecture, Institute of Computing Technology, Chinese Academy of Science, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SKL of Computer Architecture, Institute of Computing Technology, Chinese Academy of Science, Beijing, China","institution_ids":["https://openalex.org/I4210090176"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.2389,"has_fulltext":false,"cited_by_count":22,"citation_normalized_percentile":{"value":0.89694459,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"32","issue":"7","first_page":"1702","last_page":"1712"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8629615902900696},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7089787721633911},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6646302342414856},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.615459680557251},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5350853800773621},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.49333158135414124},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.48174619674682617},{"id":"https://openalex.org/keywords/asynchronous-communication","display_name":"Asynchronous communication","score":0.45587390661239624},{"id":"https://openalex.org/keywords/parallel-algorithm","display_name":"Parallel algorithm","score":0.414798378944397},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.18266448378562927}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8629615902900696},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7089787721633911},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6646302342414856},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.615459680557251},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5350853800773621},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.49333158135414124},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.48174619674682617},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.45587390661239624},{"id":"https://openalex.org/C120373497","wikidata":"https://www.wikidata.org/wiki/Q1087987","display_name":"Parallel algorithm","level":2,"score":0.414798378944397},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.18266448378562927},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2020.3048836","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2020.3048836","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1982466500","display_name":"\u9762\u5411\u4f17\u6838\u4f53\u7cfb\u67b6\u6784\u7684\u5e76\u884c\u8ba1\u7b97\u6a21\u578b\u4e0e\u6027\u80fd\u81ea\u9002\u5e94\u4f18\u5316\u7814\u7a76","funder_award_id":"61972376","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2849673663","display_name":"\u9762\u5411\u6c14\u5019\u548c\u6e4d\u6d41\u6a21\u62df\u7684\u767e\u4e07\u91cf\u7ea7\u5f02\u6784\u4f17\u6838\u53ef\u6269\u5c55\u5e76\u884c\u7b97\u6cd5\u4e0e\u4f18\u5316\u65b9\u6cd5","funder_award_id":"61432018","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4010769507","display_name":null,"funder_award_id":"L182053","funder_id":"https://openalex.org/F4320322919","funder_display_name":"Natural Science Foundation of Beijing Municipality"},{"id":"https://openalex.org/G6244656024","display_name":null,"funder_award_id":"61502450","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8380396287","display_name":null,"funder_award_id":"61521092","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322919","display_name":"Natural Science Foundation of Beijing Municipality","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":77,"referenced_works":["https://openalex.org/W114517082","https://openalex.org/W1572403558","https://openalex.org/W1603765807","https://openalex.org/W1616857247","https://openalex.org/W1667652561","https://openalex.org/W1939652453","https://openalex.org/W1946315329","https://openalex.org/W1992208280","https://openalex.org/W2023411705","https://openalex.org/W2106709023","https://openalex.org/W2118545728","https://openalex.org/W2120293976","https://openalex.org/W2123154536","https://openalex.org/W2130062883","https://openalex.org/W2133233009","https://openalex.org/W2138243089","https://openalex.org/W2143021454","https://openalex.org/W2146502635","https://openalex.org/W2155893237","https://openalex.org/W2166706236","https://openalex.org/W2186615578","https://openalex.org/W2199097987","https://openalex.org/W2336051946","https://openalex.org/W2402144811","https://openalex.org/W2405601855","https://openalex.org/W2517617279","https://openalex.org/W2622263826","https://openalex.org/W2749988060","https://openalex.org/W2783257164","https://openalex.org/W2787017828","https://openalex.org/W2808079723","https://openalex.org/W2889676205","https://openalex.org/W2895512264","https://openalex.org/W2930786691","https://openalex.org/W2950002113","https://openalex.org/W2951781666","https://openalex.org/W2952046647","https://openalex.org/W2952594493","https://openalex.org/W2962747323","https://openalex.org/W2963839817","https://openalex.org/W2963861706","https://openalex.org/W2963927126","https://openalex.org/W2967558351","https://openalex.org/W2970989202","https://openalex.org/W2979364645","https://openalex.org/W2983101505","https://openalex.org/W2988070836","https://openalex.org/W2988736552","https://openalex.org/W3003166701","https://openalex.org/W3034241250","https://openalex.org/W3040257343","https://openalex.org/W3099748883","https://openalex.org/W3106152900","https://openalex.org/W4288023687","https://openalex.org/W4302296459","https://openalex.org/W6637151318","https://openalex.org/W6640486635","https://openalex.org/W6675763415","https://openalex.org/W6678044193","https://openalex.org/W6678379972","https://openalex.org/W6679314259","https://openalex.org/W6681283895","https://openalex.org/W6681435938","https://openalex.org/W6684249991","https://openalex.org/W6686509673","https://openalex.org/W6703506510","https://openalex.org/W6739622702","https://openalex.org/W6743289643","https://openalex.org/W6748108687","https://openalex.org/W6748296962","https://openalex.org/W6752615867","https://openalex.org/W6754529011","https://openalex.org/W6760810938","https://openalex.org/W6763775535","https://openalex.org/W6764309476","https://openalex.org/W6764647576","https://openalex.org/W6770532375"],"related_works":["https://openalex.org/W2168758875","https://openalex.org/W2410733619","https://openalex.org/W4246549241","https://openalex.org/W2963483475","https://openalex.org/W1876383975","https://openalex.org/W2499279132","https://openalex.org/W2607998022","https://openalex.org/W2059382600","https://openalex.org/W2005006850","https://openalex.org/W2357914513"],"abstract_inverted_index":{"As":[0],"the":[1,6,21,25,49,53,60,82,109,117,127,133,143,150],"training":[2,22,102,147,157],"dataset":[3,54,74],"size":[4,8],"and":[5,27,91,131],"model":[7,119],"of":[9,30,62,73,84,100,146,154],"machine":[10,32,64],"learning":[11,33,65],"increase":[12],"rapidly,":[13],"more":[14],"computing":[15],"resources":[16],"are":[17,41],"consumed":[18],"to":[19,69,75],"speedup":[20],"process.":[23],"However,":[24],"scalability":[26,61,151],"performance":[28],"reproducibility":[29],"parallel":[31,63,101,111,118,156],"training,":[34],"which":[35],"mainly":[36],"uses":[37],"stochastic":[38],"optimization":[39,129,136],"algorithms,":[40],"limited.":[42],"In":[43],"this":[44],"paper,":[45],"we":[46],"demonstrate":[47],"that":[48,142],"sample":[50,77,85,87,89],"difference":[51],"in":[52,59,93],"plays":[55],"a":[56],"prominent":[57],"role":[58],"algorithms.":[66,158],"We":[67,96],"propose":[68],"use":[70],"statistical":[71,144],"properties":[72,80,145],"measure":[76],"differences.":[78],"These":[79],"include":[81],"variance":[83],"features,":[86],"sparsity,":[88],"diversity,":[90],"similarity":[92],"sampling":[94],"sequences.":[95],"choose":[97],"four":[98],"types":[99],"algorithms":[103],"as":[104],"our":[105],"research":[106],"objects:":[107],"(1)":[108],"asynchronous":[110],"SGD":[112,121,124],"algorithm":[113,122],"(Hogwild!":[114],"algorithm),":[115,125],"(2)":[116],"average":[120],"(minibatch":[123],"(3)":[126],"decentralization":[128],"algorithm,":[130],"(4)":[132],"dual":[134],"coordinate":[135],"(DADM":[137],"algorithm).":[138],"Our":[139],"results":[140],"show":[141],"datasets":[148],"determine":[149],"upper":[152],"bound":[153],"these":[155]},"counts_by_year":[{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}