{"id":"https://openalex.org/W4404400729","doi":"https://doi.org/10.1145/3694715.3695969","title":"Enabling Parallelism Hot Switching for Efficient Training of Large Language Models","display_name":"Enabling Parallelism Hot Switching for Efficient Training of Large Language Models","publication_year":2024,"publication_date":"2024-11-04","ids":{"openalex":"https://openalex.org/W4404400729","doi":"https://doi.org/10.1145/3694715.3695969"},"language":"en","primary_location":{"id":"doi:10.1145/3694715.3695969","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3694715.3695969","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3694715.3695969","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM SIGOPS 30th Symposium on Operating Systems Principles","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3694715.3695969","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5108863265","display_name":"Houyang Ge","orcid":"https://orcid.org/0009-0001-3367-7486"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hao Ge","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0001-3367-7486","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039254679","display_name":"Fangcheng Fu","orcid":"https://orcid.org/0000-0003-1658-0380"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fangcheng Fu","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1658-0380","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029750339","display_name":"Haoyang Li","orcid":"https://orcid.org/0009-0001-5342-0194"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haoyang Li","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0001-5342-0194","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Xuanyu Wang","orcid":"https://orcid.org/0009-0006-6458-7033"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuanyu Wang","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0006-6458-7033","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111914292","display_name":"Sheng Lin","orcid":"https://orcid.org/0009-0002-1495-0499"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sheng Lin","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0002-1495-0499","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103276362","display_name":"Y X Wang","orcid":"https://orcid.org/0009-0000-8375-493X"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yujie Wang","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0000-8375-493X","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059601307","display_name":"Xiaonan Nie","orcid":"https://orcid.org/0000-0001-6766-757X"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaonan Nie","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-6766-757X","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100462441","display_name":"Hailin Zhang","orcid":"https://orcid.org/0009-0000-4188-7742"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hailin Zhang","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0000-4188-7742","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015552951","display_name":"Xupeng Miao","orcid":"https://orcid.org/0000-0002-9371-8358"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xupeng Miao","raw_affiliation_strings":["Purdue University, West Lafayette, USA"],"raw_orcid":"https://orcid.org/0000-0002-9371-8358","affiliations":[{"raw_affiliation_string":"Purdue University, West Lafayette, USA","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062357883","display_name":"Bin Cui","orcid":"https://orcid.org/0000-0003-1681-4677"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Cui","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1681-4677","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.4224,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.84593507,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"178","last_page":"194"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.984000027179718,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9811000227928162,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8113313913345337},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.7569361925125122},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.6023558378219604},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5534098148345947},{"id":"https://openalex.org/keywords/data-parallelism","display_name":"Data parallelism","score":0.4786325991153717},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4302135109901428},{"id":"https://openalex.org/keywords/task-parallelism","display_name":"Task parallelism","score":0.42974066734313965},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.37182778120040894}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8113313913345337},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.7569361925125122},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6023558378219604},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5534098148345947},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.4786325991153717},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4302135109901428},{"id":"https://openalex.org/C42992933","wikidata":"https://www.wikidata.org/wiki/Q691169","display_name":"Task parallelism","level":3,"score":0.42974066734313965},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.37182778120040894},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3694715.3695969","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3694715.3695969","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3694715.3695969","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM SIGOPS 30th Symposium on Operating Systems Principles","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3694715.3695969","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3694715.3695969","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3694715.3695969","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM SIGOPS 30th Symposium on Operating Systems Principles","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.47999998927116394,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G1691401027","display_name":null,"funder_award_id":"U22B2037","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3648843532","display_name":null,"funder_award_id":"U23B2048","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4153579485","display_name":null,"funder_award_id":"SH-2024JK29","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8136954100","display_name":null,"funder_award_id":"2022ZD0116315","funder_id":"https://openalex.org/F4320329860","funder_display_name":"National Science and Technology Major Project"}],"funders":[{"id":"https://openalex.org/F4320316083","display_name":"Tencent","ror":"https://ror.org/00hhjss72"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321543","display_name":"China Postdoctoral Science Foundation","ror":"https://ror.org/0426zh255"},{"id":"https://openalex.org/F4320324787","display_name":"Peking University","ror":"https://ror.org/02v51f717"},{"id":"https://openalex.org/F4320329860","display_name":"National Science and Technology Major Project","ror":null},{"id":"https://openalex.org/F4320335768","display_name":"National Postdoctoral Program for Innovative Talents","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404400729.pdf","grobid_xml":"https://content.openalex.org/works/W4404400729.grobid-xml"},"referenced_works_count":23,"referenced_works":["https://openalex.org/W398859631","https://openalex.org/W569478347","https://openalex.org/W2489529491","https://openalex.org/W2799200478","https://openalex.org/W2969388332","https://openalex.org/W2981758446","https://openalex.org/W2981852735","https://openalex.org/W3004659153","https://openalex.org/W3019389960","https://openalex.org/W3081168214","https://openalex.org/W3086105743","https://openalex.org/W4214708455","https://openalex.org/W4224308101","https://openalex.org/W4230315356","https://openalex.org/W4243417494","https://openalex.org/W4288089799","https://openalex.org/W4301361180","https://openalex.org/W4310282800","https://openalex.org/W4362566358","https://openalex.org/W4362566383","https://openalex.org/W4386768656","https://openalex.org/W4386768680","https://openalex.org/W6810081322"],"related_works":["https://openalex.org/W2950520577","https://openalex.org/W1501159154","https://openalex.org/W2003935582","https://openalex.org/W74409296","https://openalex.org/W1554644772","https://openalex.org/W2494130044","https://openalex.org/W2468095077","https://openalex.org/W29548032","https://openalex.org/W4230999561","https://openalex.org/W1595672120"],"abstract_inverted_index":{"Training":[0],"of":[1,18],"large-scale":[2],"deep":[3],"learning":[4],"models":[5,67],"necessitates":[6],"parallelizing":[7],"the":[8,16,24,76],"model":[9],"and":[10,15,33],"data":[11],"across":[12,43,80],"numerous":[13],"devices,":[14],"choice":[17],"parallelism":[19,58,87],"strategy":[20,59,88],"substantially":[21],"depends":[22],"on":[23],"training":[25,41,62,66,83],"workloads":[26,42],"such":[27],"as":[28],"memory":[29],"consumption,":[30],"computation":[31],"cost,":[32],"communication":[34],"cost.":[35],"Current":[36],"approaches":[37],"generally":[38],"assume":[39],"uniform":[40],"samples":[44],"in":[45,91],"a":[46,56,85],"given":[47],"task.":[48],"Thus,":[49],"existing":[50],"systems":[51],"are":[52],"designed":[53],"to":[54,75],"adopt":[55],"static":[57,86],"throughout":[60],"one":[61],"process.":[63],"Nevertheless,":[64],"when":[65],"with":[68,84],"sequence":[69,77],"inputs,":[70],"this":[71],"assumption":[72],"fails":[73],"due":[74],"length":[78],"variation":[79],"samples.":[81],"Consequently,":[82],"would":[89],"result":[90],"sub-optimal":[92],"performance.":[93]},"counts_by_year":[{"year":2025,"cited_by_count":4}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}