{"id":"https://openalex.org/W7131823304","doi":"https://doi.org/10.48550/arxiv.2602.22437","title":"veScale-FSDP: Flexible and High-Performance FSDP at Scale","display_name":"veScale-FSDP: Flexible and High-Performance FSDP at Scale","publication_year":2026,"publication_date":"2026-02-25","ids":{"openalex":"https://openalex.org/W7131823304","doi":"https://doi.org/10.48550/arxiv.2602.22437"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2602.22437","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.22437","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2602.22437","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5127409623","display_name":"Zezhou Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Wang, Zezhou","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121357151","display_name":"Youjie Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Youjie","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127283434","display_name":"Zhiqi Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Zhiqi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127401924","display_name":"Jiacheng Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Jiacheng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127279464","display_name":"Cong Xie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xie, Cong","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127187023","display_name":"Guanyu Feng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feng, Guanyu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127177153","display_name":"Zheng Zhong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhong, Zheng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127304397","display_name":"Ziyue Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Ziyue","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127439736","display_name":"Hongyu Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Hongyu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127095384","display_name":"Zhi Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Zhi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127089434","display_name":"Yanghua Peng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Yanghua","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5127103450","display_name":"Xin Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xin","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":12,"corresponding_author_ids":["https://openalex.org/A5127409623"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.4147000014781952,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.4147000014781952,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.3431999981403351,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.026799999177455902,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.6949999928474426},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.6190999746322632},{"id":"https://openalex.org/keywords/flexibility","display_name":"Flexibility (engineering)","score":0.5909000039100647},{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.49390000104904175},{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.4724999964237213},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.4357999861240387},{"id":"https://openalex.org/keywords/macro","display_name":"Macro","score":0.39579999446868896}],"concepts":[{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.6949999928474426},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.65420001745224},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.6190999746322632},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.5909000039100647},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.49390000104904175},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.4724999964237213},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.4357999861240387},{"id":"https://openalex.org/C166955791","wikidata":"https://www.wikidata.org/wiki/Q629579","display_name":"Macro","level":2,"score":0.39579999446868896},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3806000053882599},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.3716000020503998},{"id":"https://openalex.org/C75553542","wikidata":"https://www.wikidata.org/wiki/Q178161","display_name":"A priori and a posteriori","level":2,"score":0.357699990272522},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.35659998655319214},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.34929999709129333},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.34290000796318054},{"id":"https://openalex.org/C162319229","wikidata":"https://www.wikidata.org/wiki/Q175263","display_name":"Data structure","level":2,"score":0.32269999384880066},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.3041999936103821},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.2856999933719635},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.25619998574256897},{"id":"https://openalex.org/C158251709","wikidata":"https://www.wikidata.org/wiki/Q354025","display_name":"Intrusion","level":2,"score":0.2556999921798706}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2602.22437","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.22437","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2602.22437","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.22437","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Fully":[0],"Sharded":[1],"Data":[2],"Parallel":[3],"(FSDP),":[4],"also":[5],"known":[6],"as":[7],"ZeRO,":[8],"is":[9],"widely":[10],"used":[11,46],"for":[12],"training":[13,32],"large-scale":[14],"models,":[15],"featuring":[16],"its":[17],"flexibility":[18,107],"and":[19,38,44,74,108,124,135],"minimal":[20],"intrusion":[21],"on":[22],"model":[23],"code.":[24],"However,":[25],"current":[26],"FSDP":[27,90,142],"systems":[28],"struggle":[29],"with":[30,39,62,99],"structure-aware":[31,101],"methods":[33],"(e.g.,":[34,42,50],"block-wise":[35,122],"quantized":[36],"training)":[37],"non-element-wise":[40,125],"optimizers":[41],"Shampoo":[43],"Muon)":[45],"in":[47,72],"cutting-edge":[48],"models":[49],"Gemini,":[51],"Kimi":[52],"K2).":[53],"FSDP's":[54],"fixed":[55],"element-":[56],"or":[57],"row-wise":[58],"sharding":[59,96],"formats":[60],"conflict":[61],"the":[63],"block-structured":[64],"computations.":[65],"In":[66],"addition,":[67],"today's":[68],"implementations":[69],"fall":[70],"short":[71],"communication":[73],"memory":[75,138],"efficiency,":[76],"limiting":[77],"scaling":[78,145],"to":[79,104,147],"tens":[80,148],"of":[81,83,149,151],"thousands":[82,150],"GPUs.":[84,152],"We":[85],"introduce":[86],"veScale-FSDP,":[87],"a":[88,94,100,128],"redesigned":[89],"system":[91],"that":[92],"couples":[93],"flexible":[95],"format,":[97],"RaggedShard,":[98],"planning":[102],"algorithm":[103],"deliver":[105],"both":[106],"performance":[109],"at":[110],"scale.":[111],"veScale-FSDP":[112,130],"natively":[113],"supports":[114],"efficient":[115],"data":[116],"placement":[117],"required":[118],"by":[119],"FSDP,":[120],"empowering":[121],"quantization":[123],"optimizers.":[126],"As":[127],"result,":[129],"achieves":[131],"5~66%":[132],"higher":[133],"throughput":[134],"16~30%":[136],"lower":[137],"usage":[139],"than":[140],"existing":[141],"systems,":[143],"while":[144],"efficiently":[146]},"counts_by_year":[],"updated_date":"2026-02-28T06:18:59.386488","created_date":"2026-02-28T00:00:00"}