{"id":"https://openalex.org/W4392207935","doi":"https://doi.org/10.1109/tkde.2024.3370614","title":"Improving Automatic Parallel Training via Balanced Memory Workload Optimization","display_name":"Improving Automatic Parallel Training via Balanced Memory Workload Optimization","publication_year":2024,"publication_date":"2024-02-27","ids":{"openalex":"https://openalex.org/W4392207935","doi":"https://doi.org/10.1109/tkde.2024.3370614"},"language":"en","primary_location":{"id":"doi:10.1109/tkde.2024.3370614","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2024.3370614","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103276362","display_name":"Y X Wang","orcid":"https://orcid.org/0009-0000-8375-493X"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yujie Wang","raw_affiliation_strings":["School of CS and Key Lab of High Confidence Software Technologies (MOE), Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0000-8375-493X","affiliations":[{"raw_affiliation_string":"School of CS and Key Lab of High Confidence Software Technologies (MOE), Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070725905","display_name":"Youhe Jiang","orcid":"https://orcid.org/0009-0000-9297-642X"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Youhe Jiang","raw_affiliation_strings":["School of CS and Key Lab of High Confidence Software Technologies (MOE), Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0000-9297-642X","affiliations":[{"raw_affiliation_string":"School of CS and Key Lab of High Confidence Software Technologies (MOE), Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015552951","display_name":"Xupeng Miao","orcid":"https://orcid.org/0000-0002-9371-8358"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xupeng Miao","raw_affiliation_strings":["Computer Science Department, Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0000-0002-9371-8358","affiliations":[{"raw_affiliation_string":"Computer Science Department, Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039254679","display_name":"Fangcheng Fu","orcid":"https://orcid.org/0000-0003-1658-0380"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fangcheng Fu","raw_affiliation_strings":["School of CS and Key Lab of High Confidence Software Technologies (MOE), Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1658-0380","affiliations":[{"raw_affiliation_string":"School of CS and Key Lab of High Confidence Software Technologies (MOE), Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016753128","display_name":"Shenhan Zhu","orcid":"https://orcid.org/0009-0004-0267-775X"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shenhan Zhu","raw_affiliation_strings":["School of CS and Key Lab of High Confidence Software Technologies (MOE), Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0004-0267-775X","affiliations":[{"raw_affiliation_string":"School of CS and Key Lab of High Confidence Software Technologies (MOE), Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059601307","display_name":"Xiaonan Nie","orcid":"https://orcid.org/0000-0001-6766-757X"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaonan Nie","raw_affiliation_strings":["School of CS and Key Lab of High Confidence Software Technologies (MOE), Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-6766-757X","affiliations":[{"raw_affiliation_string":"School of CS and Key Lab of High Confidence Software Technologies (MOE), Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034300662","display_name":"Yaofeng Tu","orcid":"https://orcid.org/0000-0002-2616-2273"},"institutions":[{"id":"https://openalex.org/I4210098582","display_name":"ZTE (China)","ror":"https://ror.org/00rjhhq63","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210098582"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yaofeng Tu","raw_affiliation_strings":["ZTE Company, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-2616-2273","affiliations":[{"raw_affiliation_string":"ZTE Company, Beijing, China","institution_ids":["https://openalex.org/I4210098582"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062357883","display_name":"Bin Cui","orcid":"https://orcid.org/0000-0003-1681-4677"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Cui","raw_affiliation_strings":["School of CS and Key Lab of High Confidence Software Technologies (MOE), Peking University, Beijing, China","Institute of Computational Social Science, Peking University (Qingdao), China"],"raw_orcid":"https://orcid.org/0000-0003-1681-4677","affiliations":[{"raw_affiliation_string":"School of CS and Key Lab of High Confidence Software Technologies (MOE), Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Institute of Computational Social Science, Peking University (Qingdao), China","institution_ids":["https://openalex.org/I20231570"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2495,"currency":"USD","value_usd":2495},"apc_paid":null,"fwci":4.0806,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.94673768,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"36","issue":"8","first_page":"3906","last_page":"3920"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.991100013256073,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12292","display_name":"Graph Theory and Algorithms","score":0.9908999800682068,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8498286008834839},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.6712961196899414},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5448018908500671},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.43610766530036926},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.24021929502487183}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8498286008834839},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.6712961196899414},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5448018908500671},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.43610766530036926},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.24021929502487183},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tkde.2024.3370614","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2024.3370614","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4399999976158142,"display_name":"Affordable and clean energy","id":"https://metadata.un.org/sdg/7"}],"awards":[{"id":"https://openalex.org/G1691401027","display_name":null,"funder_award_id":"U22B2037","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3648843532","display_name":null,"funder_award_id":"U23B2048","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W2883830791","https://openalex.org/W2896457183","https://openalex.org/W2964110616","https://openalex.org/W2965373594","https://openalex.org/W2969388332","https://openalex.org/W2984100107","https://openalex.org/W2991040477","https://openalex.org/W3001279689","https://openalex.org/W3019389960","https://openalex.org/W3021234081","https://openalex.org/W3034999214","https://openalex.org/W3081168214","https://openalex.org/W3086105743","https://openalex.org/W3101104221","https://openalex.org/W3129831491","https://openalex.org/W3132107458","https://openalex.org/W3138516171","https://openalex.org/W3163465001","https://openalex.org/W3169622372","https://openalex.org/W3177263144","https://openalex.org/W3188065709","https://openalex.org/W3204998121","https://openalex.org/W4221150592","https://openalex.org/W4221167110","https://openalex.org/W4282576620","https://openalex.org/W4283073926","https://openalex.org/W4288089799","https://openalex.org/W4293078876","https://openalex.org/W4310282800","https://openalex.org/W4311209912","https://openalex.org/W4353071157","https://openalex.org/W4385245566","https://openalex.org/W6684249991","https://openalex.org/W6684859321","https://openalex.org/W6747917562","https://openalex.org/W6753584990","https://openalex.org/W6766673545","https://openalex.org/W6766978945","https://openalex.org/W6769275798","https://openalex.org/W6769627184","https://openalex.org/W6772383348","https://openalex.org/W6776671032","https://openalex.org/W6778883912","https://openalex.org/W6779863968","https://openalex.org/W6784333009","https://openalex.org/W6790978476","https://openalex.org/W6791353385","https://openalex.org/W6795581714","https://openalex.org/W6796768152","https://openalex.org/W6810296985","https://openalex.org/W6810764171","https://openalex.org/W6811928498","https://openalex.org/W6847515151","https://openalex.org/W6929333568"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2000785801","https://openalex.org/W986318368","https://openalex.org/W2384410913","https://openalex.org/W2352878646","https://openalex.org/W2004734601","https://openalex.org/W2130149817","https://openalex.org/W2990194547"],"abstract_inverted_index":{"Transformer":[0,150],"models":[1,31,151],"have":[2],"emerged":[3],"as":[4,17],"the":[5,18,41,86,123,153],"leading":[6],"approach":[7,104],"for":[8,20,105],"achieving":[9],"state-of-the-art":[10],"performance":[11],"across":[12,32],"various":[13],"application":[14],"domains,":[15],"serving":[16],"foundation":[19],"advanced":[21],"large-scale":[22],"deep":[23],"learning":[24],"(DL)":[25],"models.":[26],"However,":[27],"efficiently":[28],"training":[29,56,160],"these":[30],"multiple":[33,79],"GPUs":[34],"remains":[35],"a":[36,63,73,101,116,137],"complex":[37],"challenge":[38],"due":[39],"to":[40,53,62,121,127],"abundance":[42],"of":[43,155],"parallelism":[44,60,81,90,183],"options.":[45],"Existing":[46],"DL":[47],"systems":[48],"either":[49],"require":[50],"manual":[51],"efforts":[52],"design":[54],"distributed":[55,159],"plans":[57],"or":[58],"limit":[59],"combinations":[61],"constrained":[64],"search":[65,97,119],"space.":[66],"In":[67],"this":[68,95],"paper,":[69],"we":[70,99,135],"present":[71],"Galvatron-BMW,":[72],"novel":[74],"system":[75,133,174],"framework":[76],"that":[77,141,179],"integrates":[78],"prevalent":[80],"dimensions":[82],"and":[83,107,131],"automatically":[84],"identifies":[85],"most":[87],"efficient":[88],"hybrid":[89],"strategy.":[91],"To":[92],"effectively":[93],"navigate":[94],"vast":[96],"space,":[98],"employ":[100],"decision":[102],"tree":[103],"decomposition":[106],"pruning":[108],"based":[109],"on":[110,143,148,181],"intuitive":[111],"insights.":[112],"We":[113],"further":[114],"utilize":[115],"dynamic":[117],"programming":[118],"algorithm":[120],"derive":[122],"optimal":[124],"plan.":[125],"Moreover,":[126],"improve":[128],"resource":[129],"utilization":[130],"enhance":[132],"efficiency,":[134],"propose":[136],"bi-objective":[138],"optimization":[139],"workflow":[140],"focuses":[142],"workload":[144],"balance.":[145],"Our":[146],"evaluations":[147],"different":[149],"demonstrate":[152],"capabilities":[154],"Galvatron-BMW":[156,170],"in":[157],"automating":[158],"under":[161],"varying":[162],"GPU":[163],"memory":[164],"constraints.":[165],"Across":[166],"all":[167],"tested":[168],"scenarios,":[169],"consistently":[171],"achieves":[172],"superior":[173],"throughput,":[175],"surpassing":[176],"previous":[177],"approaches":[178],"rely":[180],"limited":[182],"strategies.":[184]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":11}],"updated_date":"2026-08-28T12:50:07.497085","created_date":"2024-02-28T00:00:00"}