{"id":"https://openalex.org/W4408846465","doi":"https://doi.org/10.1145/3689031.3717461","title":"Mist: Efficient Distributed Training of Large Language Models via Memory-Parallelism Co-Optimization","display_name":"Mist: Efficient Distributed Training of Large Language Models via Memory-Parallelism Co-Optimization","publication_year":2025,"publication_date":"2025-03-26","ids":{"openalex":"https://openalex.org/W4408846465","doi":"https://doi.org/10.1145/3689031.3717461"},"language":"en","primary_location":{"id":"doi:10.1145/3689031.3717461","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3689031.3717461","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twentieth European Conference on Computer Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2503.19050","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101154638","display_name":"Zhanda Zhu","orcid":"https://orcid.org/0000-0001-7433-7384"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Zhanda Zhu","raw_affiliation_strings":["University of Toronto, Vector, Institute, CentML"],"affiliations":[{"raw_affiliation_string":"University of Toronto, Vector, Institute, CentML","institution_ids":["https://openalex.org/I4210127509","https://openalex.org/I185261750"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044162748","display_name":"Christina Giannoula","orcid":"https://orcid.org/0000-0003-0162-4547"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Christina Giannoula","raw_affiliation_strings":["University of Toronto, Vector, Institute, CentML"],"affiliations":[{"raw_affiliation_string":"University of Toronto, Vector, Institute, CentML","institution_ids":["https://openalex.org/I4210127509","https://openalex.org/I185261750"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010431297","display_name":"Muralidhar Andoorveedu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Muralidhar Andoorveedu","raw_affiliation_strings":["CentML"],"affiliations":[{"raw_affiliation_string":"CentML","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109026381","display_name":"Qidong Su","orcid":null},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Qidong Su","raw_affiliation_strings":["University of Toronto, Vector, Institute, CentML"],"affiliations":[{"raw_affiliation_string":"University of Toronto, Vector, Institute, CentML","institution_ids":["https://openalex.org/I4210127509","https://openalex.org/I185261750"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038469427","display_name":"Karttikeya Mangalam","orcid":"https://orcid.org/0000-0002-2169-1395"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karttikeya Mangalam","raw_affiliation_strings":["SigIQ.ai"],"affiliations":[{"raw_affiliation_string":"SigIQ.ai","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053912518","display_name":"Bojian Zheng","orcid":"https://orcid.org/0000-0002-1999-2359"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Bojian Zheng","raw_affiliation_strings":["University of Toronto, Vector, Institute, CentML"],"affiliations":[{"raw_affiliation_string":"University of Toronto, Vector, Institute, CentML","institution_ids":["https://openalex.org/I4210127509","https://openalex.org/I185261750"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007585346","display_name":"Gennady Pekhimenko","orcid":"https://orcid.org/0000-0002-3839-0919"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Gennady Pekhimenko","raw_affiliation_strings":["University of Toronto, Vector, Institute, CentML"],"affiliations":[{"raw_affiliation_string":"University of Toronto, Vector, Institute, CentML","institution_ids":["https://openalex.org/I4210127509","https://openalex.org/I185261750"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5101154638"],"corresponding_institution_ids":["https://openalex.org/I185261750","https://openalex.org/I4210127509"],"apc_list":null,"apc_paid":null,"fwci":4.9698,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.94243172,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"1298","last_page":"1316"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9904999732971191,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8104687929153442},{"id":"https://openalex.org/keywords/mist","display_name":"Mist","score":0.7607574462890625},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.7027243375778198},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.6596417427062988},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.490054726600647},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.44184210896492004},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.35218721628189087}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8104687929153442},{"id":"https://openalex.org/C6831451","wikidata":"https://www.wikidata.org/wiki/Q192196","display_name":"Mist","level":2,"score":0.7607574462890625},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.7027243375778198},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.6596417427062988},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.490054726600647},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.44184210896492004},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.35218721628189087},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3689031.3717461","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3689031.3717461","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twentieth European Conference on Computer Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2503.19050","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.19050","pdf_url":"https://arxiv.org/pdf/2503.19050","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2503.19050","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.19050","pdf_url":"https://arxiv.org/pdf/2503.19050","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W398859631","https://openalex.org/W1710734607","https://openalex.org/W2008875774","https://openalex.org/W2122691518","https://openalex.org/W2295598076","https://openalex.org/W2489529491","https://openalex.org/W2489773810","https://openalex.org/W2612076670","https://openalex.org/W2734941459","https://openalex.org/W2883830791","https://openalex.org/W2914209329","https://openalex.org/W2969388332","https://openalex.org/W2979044977","https://openalex.org/W3012514909","https://openalex.org/W3037639655","https://openalex.org/W3043619075","https://openalex.org/W3081168214","https://openalex.org/W3101104221","https://openalex.org/W3129831491","https://openalex.org/W3132107458","https://openalex.org/W3145595827","https://openalex.org/W3157919170","https://openalex.org/W3204998121","https://openalex.org/W3205803342","https://openalex.org/W3206636350","https://openalex.org/W4205983429","https://openalex.org/W4206623734","https://openalex.org/W4220741164","https://openalex.org/W4229024507","https://openalex.org/W4285225959","https://openalex.org/W4301361180","https://openalex.org/W4307007621","https://openalex.org/W4310282800","https://openalex.org/W4318541647","https://openalex.org/W4318541673","https://openalex.org/W4320523450","https://openalex.org/W4327694855","https://openalex.org/W4385585365","https://openalex.org/W4386768656","https://openalex.org/W4388979610","https://openalex.org/W4394907705","https://openalex.org/W4394922822","https://openalex.org/W4394998532","https://openalex.org/W4394998892","https://openalex.org/W4395117922"],"related_works":["https://openalex.org/W2358279768","https://openalex.org/W2383291219","https://openalex.org/W2349572614","https://openalex.org/W2367041282","https://openalex.org/W2023900323","https://openalex.org/W2472801082","https://openalex.org/W2048390140","https://openalex.org/W2391634122","https://openalex.org/W1595672120","https://openalex.org/W4230999561"],"abstract_inverted_index":{"Various":[0],"parallelism,":[1,8],"such":[2],"as":[3],"data,":[4],"tensor,":[5],"and":[6,18,68,87,127,136,148,156,163,184,196],"pipeline":[7],"along":[9],"with":[10],"memory":[11,97,128],"optimizations":[12,115],"like":[13],"activation":[14],"checkpointing,":[15],"redundancy":[16],"elimination,":[17],"offloading,":[19],"have":[20],"been":[21],"proposed":[22],"to":[23,55,62,74,182,187,191],"accelerate":[24],"distributed":[25,40,90],"training":[26,41,91],"for":[27,133],"Large":[28],"Language":[29],"Models.":[30],"To":[31,77],"find":[32],"the":[33,56,64,70,142],"best":[34],"combination":[35],"of":[36,52,58,179],"these":[37,79],"techniques,":[38],"automatic":[39,89,198],"systems":[42,47],"are":[43],"proposed.":[44],"However,":[45],"existing":[46],"only":[48],"tune":[49],"a":[50,84],"subset":[51],"optimizations,":[53],"due":[54],"lack":[57],"overlap":[59,149],"awareness,":[60],"inability":[61],"navigate":[63],"vast":[65],"search":[66],"space,":[67],"ignoring":[69],"inter-microbatch":[71],"imbalance,":[72],"leading":[73],"sub-optimal":[75],"performance.":[76],"address":[78],"shortcomings,":[80],"we":[81],"propose":[82],"Mist,":[83],"memory,":[85],"overlap,":[86],"imbalance-aware":[88,138],"system":[92,194,199],"that":[93,124,174],"comprehensively":[94],"co-optimizes":[95],"all":[96],"footprint":[98],"reduction":[99],"techniques":[100],"alongside":[101],"parallelism.":[102],"Mist":[103,175],"is":[104],"based":[105],"on":[106],"three":[107],"key":[108],"ideas:":[109],"(1)":[110],"fine-grained":[111],"overlap-centric":[112],"scheduling,":[113],"orchestrating":[114],"in":[116],"an":[117,145,157,177],"overlapped":[118],"manner,":[119],"(2)":[120],"symbolic-based":[121],"performance":[122],"analysis":[123],"predicts":[125],"runtime":[126],"usage":[129],"using":[130],"symbolic":[131],"expressions":[132],"fast":[134],"tuning,":[135,140],"(3)":[137],"hierarchical":[139],"decoupling":[141],"process":[143],"into":[144],"inter-stage":[146],"imbalance":[147],"aware":[150],"Mixed":[151],"Integer":[152],"Linear":[153],"Programming":[154],"problem":[155],"intra-stage":[158],"Dual-Objective":[159],"Constrained":[160],"Optimization":[161],"problem,":[162],"connecting":[164],"them":[165],"through":[166],"Pareto":[167],"frontier":[168],"sampling.":[169],"Our":[170],"evaluation":[171],"results":[172],"show":[173],"achieves":[176],"average":[178],"1.28\u00d7":[180],"(up":[181,186],"1.73\u00d7)":[183],"1.27\u00d7":[185],"2.04\u00d7)":[188],"speedup":[189],"compared":[190],"state-of-the-art":[192,197],"manual":[193],"Megatron-LM":[195],"Aceso,":[200],"respectively.":[201]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}