{"id":"https://openalex.org/W7160993134","doi":"https://doi.org/10.48550/arxiv.2605.11215","title":"ReCoVer: Resilient LLM Pre-Training System via Fault-Tolerant Collective and Versatile Workload","display_name":"ReCoVer: Resilient LLM Pre-Training System via Fault-Tolerant Collective and Versatile Workload","publication_year":2026,"publication_date":"2026-05-11","ids":{"openalex":"https://openalex.org/W7160993134","doi":"https://doi.org/10.48550/arxiv.2605.11215"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.11215","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.11215","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.11215","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125628947","display_name":"Ziyue Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Ziyue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045756072","display_name":"Z. Y. Wang","orcid":"https://orcid.org/0000-0002-9473-0973"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zhengyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135979334","display_name":"Ruijie Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Ruijie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136019497","display_name":"Avinash Maurya","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maurya, Avinash","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136031221","display_name":"Hui Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Hui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134302724","display_name":"Paul Hovland","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hovland, Paul","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136063782","display_name":"Sheng Di","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Di, Sheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136076132","display_name":"Franck Cappello","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cappello, Franck","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136070006","display_name":"Bogdan Nicolae","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nicolae, Bogdan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136073209","display_name":"Zheng Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Zheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.31540000438690186,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.31540000438690186,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10772","display_name":"Distributed systems and fault tolerance","score":0.1941000074148178,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.14640000462532043,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.7278000116348267},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.6200000047683716},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.612500011920929},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.5981000065803528},{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.5573999881744385},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.5317000150680542},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.42820000648498535}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8097000122070312},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.7278000116348267},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.6200000047683716},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.612500011920929},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.5981000065803528},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.5573999881744385},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.5317000150680542},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5246000289916992},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.42820000648498535},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.38179999589920044},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.3402000069618225},{"id":"https://openalex.org/C106515295","wikidata":"https://www.wikidata.org/wiki/Q26806595","display_name":"Parallel processing","level":2,"score":0.3269999921321869},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.3215999901294708},{"id":"https://openalex.org/C78766204","wikidata":"https://www.wikidata.org/wiki/Q555032","display_name":"Multi-core processor","level":2,"score":0.3001999855041504},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.27900001406669617},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.2721000015735626},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.26429998874664307},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2590000033378601},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.257099986076355}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.11215","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.11215","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.11215","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.11215","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.811302661895752}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Pre-training":[0],"large":[1],"language":[2],"models":[3],"on":[4,28,138],"massive":[5],"GPU":[6],"clusters":[7],"has":[8],"made":[9],"hardware":[10],"faults":[11,86],"routine":[12],"rather":[13],"than":[14],"rare,":[15],"driving":[16],"the":[17,57,112,150,164,193,197],"need":[18],"for":[19,142],"resilient":[20,45],"training":[21,39,151,198],"systems.":[22],"Yet":[23],"existing":[24],"frameworks":[25],"either":[26],"focus":[27],"specific":[29],"parallelism":[30,123],"schemes":[31],"or":[32],"risk":[33],"drifting":[34],"away":[35],"from":[36,87,153],"a":[37,44,51,69,131,154],"failure-free":[38,70,155],"trajectory.":[40],"We":[41,134],"propose":[42],"ReCoVer,":[43],"LLM":[46],"pre-training":[47,140],"system":[48],"that":[49,84,95,106],"upholds":[50],"single":[52],"invariant:":[53],"each":[54],"iteration":[55],"keeps":[56],"number":[58],"of":[59,158],"microbatches":[60],"constant,":[61],"ensuring":[62],"per-iteration":[63],"gradients":[64],"remain":[65],"stochastically":[66],"equivalent":[67],"to":[68,144],"run.":[71,165],"The":[72,114],"framework":[73],"is":[74,116],"organized":[75],"as":[76,130,196],"three":[77],"decoupled":[78],"protocol":[79],"layers:":[80],"(1)":[81],"Fault-tolerant":[82],"collectives":[83],"isolate":[85],"propagating":[88],"across":[89,111,163],"replicas;":[90],"(2)":[91],"in-step":[92],"fine-grained":[93],"recovery":[94],"preserves":[96,149],"intra-iteration":[97],"progress":[98],"and":[99,124],"prevents":[100],"gradient":[101],"corruption;":[102],"(3)":[103],"versatile-workload":[104],"policy":[105],"dynamically":[107],"redistributes":[108],"microbatch":[109],"quotas":[110],"survivors.":[113],"design":[115],"parallelism-agnostic,":[117],"integrating":[118],"directly":[119],"with":[120,168,192],"both":[121],"3D":[122],"Hybrid":[125],"Sharded":[126],"Data":[127],"Parallel":[128],"(HSDP)":[129],"drop-in":[132],"substrate.":[133],"evaluate":[135],"our":[136],"implementation":[137],"end-to-end":[139],"tasks":[141],"up":[143],"512":[145],"GPUs,":[146],"ReCoVer":[147,171,184],"successfully":[148],"trajectory":[152],"reference":[156],"despite":[157],"256":[159],"GPUs":[160],"lost":[161],"spread":[162],"For":[166],"comparison":[167],"checkpoint-and-restart":[169],"baselines,":[170],"demonstrates":[172],"$2.23\\times$":[173],"higher":[174],"effective":[175],"throughput":[176],"after":[177],"successive":[178],"failures.":[179],"This":[180],"advantage":[181],"results":[182],"in":[183],"processing":[185],"74.9%":[186],"more":[187],"tokens":[188],"at":[189],"234":[190],"GPU-hours,":[191],"gap":[194],"widening":[195],"prolongs.":[199]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-14T00:00:00"}