{"id":"https://openalex.org/W7161686172","doi":"https://doi.org/10.48550/arxiv.2605.17879","title":"Guard: Scalable Straggler Detection and Node Health Management for Large-Scale Training","display_name":"Guard: Scalable Straggler Detection and Node Health Management for Large-Scale Training","publication_year":2026,"publication_date":"2026-05-18","ids":{"openalex":"https://openalex.org/W7161686172","doi":"https://doi.org/10.48550/arxiv.2605.17879"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.17879","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.17879","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.17879","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136499019","display_name":"Guanliang Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Guanliang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085975290","display_name":"Abhinandan Patni","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Patni, Abhinandan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090855834","display_name":"Congzhu Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Congzhu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094581007","display_name":"Zo\u00eb Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Zoe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011711668","display_name":"Jack Wittmayer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wittmayer, Jack","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106886545","display_name":"Josh Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Josh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136489231","display_name":"Ashvin Nihalani","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nihalani, Ashvin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136468231","display_name":"Binxuan Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Binxuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136484409","display_name":"Yinghong Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yinghong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136501385","display_name":"Rory Na","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Na, Rory","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136465158","display_name":"Anthony Ko","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ko, Anthony","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119968364","display_name":"Alexander Zhipa","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhipa, Alexander","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136504545","display_name":"Cong Cheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Cong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136459723","display_name":"Mi Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Mi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136474523","display_name":"Vijay Rajakumar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rajakumar, Vijay","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069910034","display_name":"Rejith George Joseph","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Joseph, Rejith George","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136456523","display_name":"Parthasarathy Govindarajen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Govindarajen, Parthasarathy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.6672999858856201,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.6672999858856201,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.10450000315904617,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11005","display_name":"Radiation Effects in Electronics","score":0.03620000183582306,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7476999759674072},{"id":"https://openalex.org/keywords/guard","display_name":"Guard (computer science)","score":0.6759999990463257},{"id":"https://openalex.org/keywords/debugging","display_name":"Debugging","score":0.6365000009536743},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.6276999711990356},{"id":"https://openalex.org/keywords/node","display_name":"Node (physics)","score":0.529699981212616},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.4162999987602234}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7767999768257141},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7476999759674072},{"id":"https://openalex.org/C141141315","wikidata":"https://www.wikidata.org/wiki/Q2379942","display_name":"Guard (computer science)","level":2,"score":0.6759999990463257},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.6365000009536743},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.6276999711990356},{"id":"https://openalex.org/C62611344","wikidata":"https://www.wikidata.org/wiki/Q1062658","display_name":"Node (physics)","level":2,"score":0.529699981212616},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.4162999987602234},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.3822999894618988},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.3747999966144562},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3580000102519989},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.33730000257492065},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.273499995470047},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.26080000400543213},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.26010000705718994}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.17879","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.17879","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.17879","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.17879","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Training":[0],"frontier-scale":[1],"foundation":[2,118],"models":[3],"involves":[4],"coordinating":[5],"tens":[6],"of":[7,9],"thousands":[8],"GPUs":[10],"over":[11],"multi-month":[12],"runs,":[13],"where":[14],"even":[15],"minor":[16],"performance":[17,76],"degradations":[18],"can":[19],"accumulate":[20],"into":[21],"substantial":[22],"efficiency":[23],"losses.":[24],"Existing":[25],"health-check":[26],"mechanisms,":[27],"such":[28],"as":[29],"NCCL":[30],"tests":[31],"or":[32],"GPU":[33],"burn-in,":[34],"primarily":[35],"focus":[36],"on":[37,116],"functional":[38],"correctness":[39],"and":[40,64,88,106,146,150,160,169],"often":[41],"fail":[42],"to":[43,101,129,138,143],"detect":[44,102],"fail-slow":[45,108],"behaviors":[46,109],"that":[47,85,110,156],"silently":[48],"degrade":[49],"system":[50,60],"performance.":[51],"In":[52],"this":[53],"paper,":[54],"we":[55],"present":[56],"Guard,":[57],"a":[58],"scalable":[59],"for":[61,166],"detecting":[62],"stragglers":[63],"ensuring":[65],"node":[66,162],"health":[67],"in":[68,94],"large-scale":[69,117,171],"training":[70,79,133],"clusters.":[71],"Guard":[72,100,122],"combines":[73],"lightweight":[74],"online":[75],"monitoring":[77],"during":[78],"with":[80],"an":[81],"offline":[82],"node-sweep":[83],"mechanism":[84],"systematically":[86],"evaluates":[87],"qualifies":[89],"nodes":[90],"before":[91],"they":[92],"participate":[93],"production":[95],"workloads.":[96],"This":[97],"design":[98],"enables":[99],"both":[103],"acute":[104],"failures":[105],"long-running":[107],"traditional":[111],"diagnostics":[112],"cannot":[113],"capture.":[114],"Deployed":[115],"model":[119],"pretraining":[120],"workloads,":[121],"improves":[123],"mean":[124,141],"FLOPs":[125],"utilization":[126],"by":[127],"up":[128],"1.7x,":[130],"reduces":[131,148],"run-to-run":[132],"step":[134],"variance":[135],"from":[136],"20%":[137],"1%,":[139],"increases":[140],"time":[142],"failure":[144],"(MTTF),":[145],"significantly":[147],"operational":[149],"debugging":[151],"overhead.":[152],"These":[153],"results":[154],"demonstrate":[155],"proactive":[157],"straggler":[158],"detection":[159],"systematic":[161],"qualification":[163],"are":[164],"critical":[165],"maintaining":[167],"stable":[168],"efficient":[170],"training.":[172]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-20T00:00:00"}