{"id":"https://openalex.org/W7133362596","doi":"https://doi.org/10.48550/arxiv.2603.00357","title":"SPARe: Stacked Parallelism with Adaptive Reordering for Fault-Tolerant LLM Pretraining Systems with 100k+ GPUs","display_name":"SPARe: Stacked Parallelism with Adaptive Reordering for Fault-Tolerant LLM Pretraining Systems with 100k+ GPUs","publication_year":2026,"publication_date":"2026-02-27","ids":{"openalex":"https://openalex.org/W7133362596","doi":"https://doi.org/10.48550/arxiv.2603.00357"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.00357","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.00357","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.00357","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5127889442","display_name":"Jin Yong Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Lee, Jin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127992104","display_name":"Zhonghao Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Zhonghao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126603478","display_name":"Xuhang He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Xuhang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128012096","display_name":"Robert Underwood","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Underwood, Robert","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121815467","display_name":"Bogdan Nicolae","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nicolae, Bogdan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128005253","display_name":"Franck Cappello","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cappello, Franck","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127936714","display_name":"Xiaoyi Lu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Xiaoyi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127995427","display_name":"Sheng Di","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Di, Sheng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5128009076","display_name":"Zheng Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Zheng","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5127889442"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.3416999876499176,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.3416999876499176,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10772","display_name":"Distributed systems and fault tolerance","score":0.3361999988555908,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.06499999761581421,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/spare-part","display_name":"Spare part","score":0.7623999714851379},{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.7376999855041504},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.6560999751091003},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.5195000171661377},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.4603999853134155},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.42320001125335693},{"id":"https://openalex.org/keywords/node","display_name":"Node (physics)","score":0.38519999384880066}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8565000295639038},{"id":"https://openalex.org/C194648553","wikidata":"https://www.wikidata.org/wiki/Q1364774","display_name":"Spare part","level":2,"score":0.7623999714851379},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.7376999855041504},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.6560999751091003},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.6032000184059143},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.5195000171661377},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.4603999853134155},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.42320001125335693},{"id":"https://openalex.org/C62611344","wikidata":"https://www.wikidata.org/wiki/Q1062658","display_name":"Node (physics)","level":2,"score":0.38519999384880066},{"id":"https://openalex.org/C137627569","wikidata":"https://www.wikidata.org/wiki/Q245471","display_name":"Cyclic redundancy check","level":3,"score":0.37540000677108765},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.36640000343322754},{"id":"https://openalex.org/C7545210","wikidata":"https://www.wikidata.org/wiki/Q838123","display_name":"Data redundancy","level":2,"score":0.3578000068664551},{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.3434999883174896},{"id":"https://openalex.org/C193702766","wikidata":"https://www.wikidata.org/wiki/Q1414548","display_name":"Concurrency","level":2,"score":0.3246999979019165},{"id":"https://openalex.org/C2775937380","wikidata":"https://www.wikidata.org/wiki/Q1232589","display_name":"Replica","level":2,"score":0.32030001282691956},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.28439998626708984},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.27459999918937683},{"id":"https://openalex.org/C174220543","wikidata":"https://www.wikidata.org/wiki/Q395307","display_name":"Rollback","level":3,"score":0.27379998564720154},{"id":"https://openalex.org/C191795146","wikidata":"https://www.wikidata.org/wiki/Q3878446","display_name":"Norm (philosophy)","level":2,"score":0.27129998803138733},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.2711000144481659}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.00357","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.00357","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.00357","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.00357","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"In":[0],"large-scale":[1],"LLM":[2],"pre-training":[3],"systems":[4],"with":[5,45,127],"100k+":[6],"GPUs,":[7,131],"failures":[8,55],"become":[9],"the":[10,14],"norm":[11],"rather":[12],"than":[13],"exception,":[15],"and":[16,67,106,115,119],"restart":[17],"costs":[18],"can":[19],"dominate":[20],"wall-clock":[21],"training":[22],"time.":[23],"However,":[24],"existing":[25],"fault-tolerance":[26,50],"mechanisms":[27],"are":[28],"largely":[29],"unprepared":[30],"for":[31,102],"this":[32,37],"restart-dominant":[33],"regime.":[34],"To":[35],"address":[36],"challenge,":[38],"we":[39],"propose":[40],"SPARe":[41,71,132],"-":[42,48],"Stacked":[43],"Parallelism":[44],"Adaptive":[46],"Reordering":[47],"a":[49],"framework":[51],"that":[52],"masks":[53],"node":[54],"during":[56],"gradient":[57],"synchronization":[58],"by":[59,135],"stacking":[60],"redundant":[61],"data":[62],"shards":[63],"across":[64],"parallelism":[65],"groups":[66],"adaptively":[68],"reordering":[69],"execution.":[70],"achieves":[72],"availability":[73],"comparable":[74],"to":[75,121,129,138],"traditional":[76,91,139],"replication":[77,92],"while":[78],"maintaining":[79],"near-constant":[80],"computation":[81,107],"overhead":[82],"of":[83],"only":[84],"2~3x,":[85],"even":[86],"under":[87],"high":[88],"redundancy":[89,118],"where":[90],"would":[93],"require":[94],"linearly":[95],"inflating":[96],"overhead.":[97],"We":[98],"derive":[99],"closed-form":[100],"expressions":[101],"endurable":[103],"failure":[104],"count":[105],"overhead,":[108],"validate":[109],"them":[110],"via":[111],"SimGrid-based":[112],"discrete-event":[113],"simulation,":[114],"jointly":[116],"optimize":[117],"checkpointing":[120],"minimize":[122],"time-to-train.":[123],"At":[124],"extreme":[125],"scale":[126],"up":[128],"600k":[130],"reduces":[133],"time-to-train":[134],"40~50%":[136],"compared":[137],"replication.":[140]},"counts_by_year":[],"updated_date":"2026-03-04T07:09:34.246503","created_date":"2026-03-04T00:00:00"}