{"id":"https://openalex.org/W4388041017","doi":"https://doi.org/10.1145/3620678.3624661","title":"Towards GPU Memory Efficiency for Distributed Training at Scale","display_name":"Towards GPU Memory Efficiency for Distributed Training at Scale","publication_year":2023,"publication_date":"2023-10-30","ids":{"openalex":"https://openalex.org/W4388041017","doi":"https://doi.org/10.1145/3620678.3624661"},"language":"en","primary_location":{"id":"doi:10.1145/3620678.3624661","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3620678.3624661","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 ACM Symposium on Cloud Computing","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5025359824","display_name":"Runxiang Cheng","orcid":"https://orcid.org/0000-0001-5058-9405"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Runxiang Cheng","raw_affiliation_strings":["University of Illinois Urbana-Champaign"],"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010054466","display_name":"Chris X. Cai","orcid":"https://orcid.org/0009-0002-3416-8939"},"institutions":[{"id":"https://openalex.org/I4210092558","display_name":"BC Platforms (Finland)","ror":"https://ror.org/002fen565","country_code":"FI","type":"company","lineage":["https://openalex.org/I4210092558"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["FI","US"],"is_corresponding":false,"raw_author_name":"Chris Cai","raw_affiliation_strings":["Meta Platforms, Inc"],"affiliations":[{"raw_affiliation_string":"Meta Platforms, Inc","institution_ids":["https://openalex.org/I4210092558","https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060417632","display_name":"S. Levent Yilmaz","orcid":"https://orcid.org/0009-0006-6071-5085"},"institutions":[{"id":"https://openalex.org/I4210092558","display_name":"BC Platforms (Finland)","ror":"https://ror.org/002fen565","country_code":"FI","type":"company","lineage":["https://openalex.org/I4210092558"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["FI","US"],"is_corresponding":false,"raw_author_name":"Selman Yilmaz","raw_affiliation_strings":["Meta Platforms, Inc"],"affiliations":[{"raw_affiliation_string":"Meta Platforms, Inc","institution_ids":["https://openalex.org/I4210092558","https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102897810","display_name":"Rahul Mitra","orcid":"https://orcid.org/0009-0009-3506-7612"},"institutions":[{"id":"https://openalex.org/I4210092558","display_name":"BC Platforms (Finland)","ror":"https://ror.org/002fen565","country_code":"FI","type":"company","lineage":["https://openalex.org/I4210092558"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["FI","US"],"is_corresponding":false,"raw_author_name":"Rahul Mitra","raw_affiliation_strings":["Meta Platforms, Inc"],"affiliations":[{"raw_affiliation_string":"Meta Platforms, Inc","institution_ids":["https://openalex.org/I4210092558","https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001214135","display_name":"Malay Bag","orcid":"https://orcid.org/0009-0001-3272-368X"},"institutions":[{"id":"https://openalex.org/I4210092558","display_name":"BC Platforms (Finland)","ror":"https://ror.org/002fen565","country_code":"FI","type":"company","lineage":["https://openalex.org/I4210092558"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["FI","US"],"is_corresponding":false,"raw_author_name":"Malay Bag","raw_affiliation_strings":["Meta Platforms, Inc"],"affiliations":[{"raw_affiliation_string":"Meta Platforms, Inc","institution_ids":["https://openalex.org/I4210092558","https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101722227","display_name":"M. N. Ghosh","orcid":"https://orcid.org/0000-0001-8515-0494"},"institutions":[{"id":"https://openalex.org/I4210092558","display_name":"BC Platforms (Finland)","ror":"https://ror.org/002fen565","country_code":"FI","type":"company","lineage":["https://openalex.org/I4210092558"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["FI","US"],"is_corresponding":false,"raw_author_name":"Mrinmoy Ghosh","raw_affiliation_strings":["Meta Platforms, Inc"],"affiliations":[{"raw_affiliation_string":"Meta Platforms, Inc","institution_ids":["https://openalex.org/I4210092558","https://openalex.org/I4210114444"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027605695","display_name":"Tianyin Xu","orcid":"https://orcid.org/0000-0003-4443-8170"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tianyin Xu","raw_affiliation_strings":["University of Illinois Urbana-Champaign"],"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5025359824"],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":null,"apc_paid":null,"fwci":0.597,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.69811025,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"281","last_page":"297"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/terabyte","display_name":"Terabyte","score":0.9320611953735352},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8552488088607788},{"id":"https://openalex.org/keywords/inefficiency","display_name":"Inefficiency","score":0.7906948328018188},{"id":"https://openalex.org/keywords/provisioning","display_name":"Provisioning","score":0.6715412139892578},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.48661431670188904},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.45481976866722107},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.3752424418926239},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.34693533182144165},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.1783980429172516}],"concepts":[{"id":"https://openalex.org/C199683683","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Terabyte","level":2,"score":0.9320611953735352},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8552488088607788},{"id":"https://openalex.org/C2778869765","wikidata":"https://www.wikidata.org/wiki/Q6028363","display_name":"Inefficiency","level":2,"score":0.7906948328018188},{"id":"https://openalex.org/C172191483","wikidata":"https://www.wikidata.org/wiki/Q1071806","display_name":"Provisioning","level":2,"score":0.6715412139892578},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.48661431670188904},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.45481976866722107},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3752424418926239},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.34693533182144165},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.1783980429172516},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3620678.3624661","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3620678.3624661","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 ACM Symposium on Cloud Computing","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.4399999976158142,"id":"https://metadata.un.org/sdg/9"}],"awards":[{"id":"https://openalex.org/G2374893308","display_name":null,"funder_award_id":"1956007","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4210137855","display_name":null,"funder_award_id":"CNS-2145295","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4755098675","display_name":"SHF: Medium: Collaborative Research: Enhancing Continuous Integration Testing for the Open-Source Ecosystem","funder_award_id":"1763788","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6024149408","display_name":"CAREER: Rethinking Configuration Management for Cloud and Datacenter Systems","funder_award_id":"2145295","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6391012684","display_name":null,"funder_award_id":"CCF-1763788","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":44,"referenced_works":["https://openalex.org/W2041783719","https://openalex.org/W2114079787","https://openalex.org/W2129542763","https://openalex.org/W2160121678","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2261611353","https://openalex.org/W2489529491","https://openalex.org/W2512971201","https://openalex.org/W2764100055","https://openalex.org/W2798515322","https://openalex.org/W2892785410","https://openalex.org/W2899071864","https://openalex.org/W2901299405","https://openalex.org/W2968594320","https://openalex.org/W2969388332","https://openalex.org/W2970971581","https://openalex.org/W3016842236","https://openalex.org/W3022548332","https://openalex.org/W3035078899","https://openalex.org/W3109610142","https://openalex.org/W3129831491","https://openalex.org/W3157306683","https://openalex.org/W3158146252","https://openalex.org/W3158783012","https://openalex.org/W3197720002","https://openalex.org/W3197816522","https://openalex.org/W3204998121","https://openalex.org/W3209366914","https://openalex.org/W3209503812","https://openalex.org/W4214658871","https://openalex.org/W4220741164","https://openalex.org/W4225966566","https://openalex.org/W4226479682","https://openalex.org/W4286307983","https://openalex.org/W4287594180","https://openalex.org/W4301361180","https://openalex.org/W4321636494","https://openalex.org/W6600175266","https://openalex.org/W6605114868","https://openalex.org/W6684859321","https://openalex.org/W6702248584","https://openalex.org/W6756379755","https://openalex.org/W6778883912"],"related_works":["https://openalex.org/W2066858118","https://openalex.org/W3011699130","https://openalex.org/W2134017072","https://openalex.org/W1976914335","https://openalex.org/W2915208987","https://openalex.org/W2264067234","https://openalex.org/W2152256925","https://openalex.org/W3124243301","https://openalex.org/W1940452713","https://openalex.org/W1571502335"],"abstract_inverted_index":{"The":[0],"scale":[1],"of":[2,17,24,42,65,67,72,82,96,100,117,125],"deep":[3],"learning":[4],"models":[5,13,26],"has":[6],"grown":[7],"tremendously":[8],"in":[9,105,162],"recent":[10],"years.":[11],"State-of-the-art":[12],"have":[14],"reached":[15],"billions":[16],"parameters":[18],"and":[19,30,57,139],"terabyte-scale":[20],"model":[21],"sizes.":[22],"Training":[23],"these":[25],"demands":[27],"memory":[28,51,55,70,98,133,137,143],"bandwidth":[29],"capacity":[31],"that":[32],"can":[33],"only":[34],"be":[35],"accommodated":[36],"distributively":[37],"over":[38],"hundreds":[39],"to":[40,85,150,155],"thousands":[41],"GPUs.":[43],"However,":[44],"large-scale":[45,101],"distributed":[46,73,102],"training":[47,74,103,115],"suffers":[48],"from":[49,123],"GPU":[50,69,97,132,136,142,159],"inefficiency,":[52],"such":[53,86],"as":[54],"under-utilization":[56],"out-of-memory":[58],"events":[59],"(OOMs).":[60],"There":[61],"is":[62,111],"a":[63,93,157],"lack":[64],"understanding":[66,154],"actual":[68],"behavior":[71,99],"on":[75,113,152],"terabyte-size":[76],"models,":[77],"which":[78],"hinders":[79],"the":[80,153],"development":[81],"effective":[83],"solutions":[84],"inefficiency.":[87],"In":[88],"this":[89],"paper,":[90],"we":[91],"present":[92],"systematic":[94],"analysis":[95,110],"jobs":[104,116],"production":[106,128],"at":[107],"Meta.":[108],"Our":[109],"based":[112],"offline":[114],"multi-terabyte":[118],"Deep":[119],"Learning":[120],"Recommendation":[121],"Models":[122],"one":[124],"Meta's":[126],"largest":[127],"clusters.":[129],"We":[130,146],"measure":[131],"inefficiency;":[134],"characterize":[135],"utilization,":[138],"provide":[140],"fine-grained":[141],"usage":[144],"analysis.":[145],"further":[147],"show":[148],"how":[149],"build":[151],"develop":[156],"practical":[158],"provisioning":[160],"system":[161],"production.":[163]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}