{"id":"https://openalex.org/W4407186908","doi":"https://doi.org/10.48550/arxiv.2502.02581","title":"Themis: Efficient Sparse Model Training Through Fully Sharded Sparse Data Parallelism","display_name":"Themis: Efficient Sparse Model Training Through Fully Sharded Sparse Data Parallelism","publication_year":2025,"publication_date":"2025-02-04","ids":{"openalex":"https://openalex.org/W4407186908","doi":"https://doi.org/10.48550/arxiv.2502.02581"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.02581","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2502.02581","pdf_url":"https://arxiv.org/pdf/2502.02581","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.02581","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083329618","display_name":"Yuhao Qing","orcid":"https://orcid.org/0009-0009-1500-2192","observed_orcids":["https://orcid.org/0009-0009-1500-2192"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Qing, Yuhao","raw_affiliation_strings":["The University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5117137726","display_name":"Guo-Yu Zhu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhu, Guichao","raw_affiliation_strings":["The University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016904555","display_name":"Fanxin Li","orcid":"https://orcid.org/0000-0003-2268-3036","observed_orcids":["https://orcid.org/0000-0003-2268-3036"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Lei, Lintian","raw_affiliation_strings":["The University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016904555","display_name":"Fanxin Li","orcid":"https://orcid.org/0000-0003-2268-3036","observed_orcids":["https://orcid.org/0000-0003-2268-3036"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Li, Fanxin","raw_affiliation_strings":["The University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148035804","display_name":"Shixiong Zhao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhao, Shixiong","raw_affiliation_strings":["The University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147998526","display_name":"Zekai Sun","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Sun, Zekai","raw_affiliation_strings":["The University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035241109","display_name":"Xiuxian Guan","orcid":"https://orcid.org/0000-0001-6133-8388","observed_orcids":["https://orcid.org/0000-0001-6133-8388"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Guan, Xiuxian","raw_affiliation_strings":["The University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101527007","display_name":"Xusheng Chen","orcid":"https://orcid.org/0000-0002-2807-9780","observed_orcids":["https://orcid.org/0000-0002-2807-9780","https://orcid.org/0009-0004-7344-7505","https://orcid.org/0009-0007-5846-013X"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Chen, Xusheng","raw_affiliation_strings":["The University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050270789","display_name":"Dong Huang","orcid":"https://orcid.org/0000-0003-3923-8828","observed_orcids":["https://orcid.org/0000-0003-3923-8828"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Huang, Dong","raw_affiliation_strings":["The University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060036172","display_name":"Ping Luo","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Ping","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147992052","display_name":"Yiming Qiu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiu, Yiming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5015360935","display_name":"Heming Cui","orcid":"https://orcid.org/0000-0001-7746-440X","observed_orcids":["https://orcid.org/0000-0001-7746-440X"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Cui, Heming","raw_affiliation_strings":["The University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I889458895"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9340999722480774,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9340999722480774,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.021400000900030136,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.011900000274181366,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mixture-of-experts","display_name":"mixture of experts","score":0.9670000076293945},{"id":"https://openalex.org/keywords/straggler-effect","display_name":"straggler effect","score":0.8320000171661377},{"id":"https://openalex.org/keywords/large-language-model-training","display_name":"large language model training","score":0.7870000004768372},{"id":"https://openalex.org/keywords/memory-usage","display_name":"memory usage","score":0.37599998712539673}],"concepts":[{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.8230975866317749},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7100119590759277},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.5862274765968323},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5762131810188293},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5116682052612305},{"id":"https://openalex.org/C124066611","wikidata":"https://www.wikidata.org/wiki/Q28684319","display_name":"Sparse approximation","level":2,"score":0.46384599804878235},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.41243240237236023},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3298869729042053},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.02581","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2502.02581","pdf_url":"https://arxiv.org/pdf/2502.02581","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.02581","doi":"https://doi.org/10.48550/arxiv.2502.02581","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.02581","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.02581","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2502.02581","pdf_url":"https://arxiv.org/pdf/2502.02581","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322724","display_name":"Ministry of Education, India","ror":"https://ror.org/048xjjh50"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4407186908.pdf","grobid_xml":"https://content.openalex.org/works/W4407186908.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2950520577","https://openalex.org/W1501159154","https://openalex.org/W1554644772","https://openalex.org/W2003935582","https://openalex.org/W2494130044","https://openalex.org/W3170887803","https://openalex.org/W74409296","https://openalex.org/W3209384898","https://openalex.org/W4230999561","https://openalex.org/W1595672120"],"abstract_inverted_index":{"Mixture-of-Experts":[0],"(MoE)":[1],"scales":[2],"large":[3],"language":[4],"models":[5],"cost-effectively,":[6],"but":[7],"expert-parallel":[8],"training":[9,51,60],"suffers":[10],"severe":[11],"straggler":[12],"effects":[13],"from":[14,112],"skewed":[15],"expert":[16,22,70],"loads.":[17],"Current":[18],"systems":[19],"frequently":[20],"rearrange":[21],"placement":[23,91,127,137],"to":[24,96,108,116],"mitigate":[25],"stragglers,":[26],"inflating":[27],"memory":[28,78,99,110,122],"footprint":[29],"and":[30,67,77,85,142],"migration":[31,75],"overhead,":[32],"potentially":[33],"negating":[34],"the":[35,63,113,117],"benefits":[36],"of":[37],"load":[38,56],"balancing.":[39],"We":[40,129],"present":[41],"Fully":[42],"Sharded":[43],"Sparse":[44],"Data":[45],"Parallelism":[46],"(FSSDP),":[47],"a":[48],"sparse-native":[49],"MoE":[50,82],"approach":[52],"that":[53],"enables":[54],"in-situ":[55],"balancing":[57,64],"on":[58],"every":[59],"iteration,":[61,93],"overlapping":[62],"with":[65,73,94,134],"computation":[66],"eliminating":[68],"explicit":[69],"rearrangement":[71],"together":[72],"its":[74],"traffic":[76],"reserves.":[79],"FSSDP":[80,102,131],"keeps":[81],"layers":[83],"sharded":[84],"sparsely":[86],"materializes":[87],"an":[88],"ephemeral,":[89],"load-balancing":[90],"each":[92],"re-materialization":[95],"reuse":[97],"available":[98],"across":[100],"layers.":[101],"is":[103],"complemented":[104],"by":[105],"heterogeneous":[106],"sharding":[107],"shift":[109],"imbalance":[111],"device":[114],"level":[115],"layer":[118],"level,":[119],"maintaining":[120],"uniform":[121],"budgets":[123],"while":[124],"enabling":[125],"per-layer":[126],"optimization.":[128],"realize":[130],"in":[132],"Themis":[133,145],"co-designed":[135],"topology-aware":[136],"algorithms.":[138],"Across":[139],"2":[140],"clusters":[141],"diverse":[143],"workloads,":[144],"achieves":[146],"1.26-2.42x":[147],"speedup":[148],"over":[149],"state-of-the-art":[150],"expert-rearrangement":[151],"systems.":[152]},"counts_by_year":[],"updated_date":"2026-09-30T09:27:27.532063","created_date":"2025-10-10T00:00:00"}