{"id":"https://openalex.org/W4408029605","doi":"https://doi.org/10.1145/3710848.3710870","title":"ATTNChecker: Highly-Optimized Fault Tolerant Attention for Large Language Model Training","display_name":"ATTNChecker: Highly-Optimized Fault Tolerant Attention for Large Language Model Training","publication_year":2025,"publication_date":"2025-02-28","ids":{"openalex":"https://openalex.org/W4408029605","doi":"https://doi.org/10.1145/3710848.3710870"},"language":"en","primary_location":{"id":"doi:10.1145/3710848.3710870","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3710848.3710870","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3710848.3710870","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM SIGPLAN Annual Symposium on Principles and Practice of Parallel Programming","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3710848.3710870","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Yuhang Liang","orcid":"https://orcid.org/0009-0008-3378-3458"},"institutions":[{"id":"https://openalex.org/I181233156","display_name":"University of Oregon","ror":"https://ror.org/0293rh119","country_code":"US","type":"education","lineage":["https://openalex.org/I181233156"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuhang Liang","raw_affiliation_strings":["University of Oregon, Oregon, USA"],"raw_orcid":"https://orcid.org/0009-0008-3378-3458","affiliations":[{"raw_affiliation_string":"University of Oregon, Oregon, USA","institution_ids":["https://openalex.org/I181233156"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063006725","display_name":"Xinyi Li","orcid":"https://orcid.org/0009-0005-7276-7715"},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xinyi Li","raw_affiliation_strings":["Pacific Northwest National Laboratory, Washington, USA"],"raw_orcid":"https://orcid.org/0009-0005-7276-7715","affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory, Washington, USA","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056482728","display_name":"Jie Ren","orcid":"https://orcid.org/0000-0001-5541-433X"},"institutions":[{"id":"https://openalex.org/I16285277","display_name":"William & Mary","ror":"https://ror.org/03hsf0573","country_code":"US","type":"education","lineage":["https://openalex.org/I16285277"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jie Ren","raw_affiliation_strings":["College of William & Mary, Virginia, USA"],"raw_orcid":"https://orcid.org/0000-0001-5541-433X","affiliations":[{"raw_affiliation_string":"College of William & Mary, Virginia, USA","institution_ids":["https://openalex.org/I16285277"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100413657","display_name":"Ang Li","orcid":"https://orcid.org/0000-0003-3734-9137"},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ang Li","raw_affiliation_strings":["Pacific Northwest National Laboratory, Washington, USA"],"raw_orcid":"https://orcid.org/0000-0003-3734-9137","affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory, Washington, USA","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036732836","display_name":"Bo Fang","orcid":null},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bo Fang","raw_affiliation_strings":["Pacific Northwest National Laboratory, Washington, USA"],"raw_orcid":"https://orcid.org/0000-0001-9721-3982","affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory, Washington, USA","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5063649910","display_name":"Jieyang Chen","orcid":"https://orcid.org/0000-0002-1905-9171"},"institutions":[{"id":"https://openalex.org/I181233156","display_name":"University of Oregon","ror":"https://ror.org/0293rh119","country_code":"US","type":"education","lineage":["https://openalex.org/I181233156"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jieyang Chen","raw_affiliation_strings":["University of Oregon, Oregon, USA"],"raw_orcid":"https://orcid.org/0000-0002-1905-9171","affiliations":[{"raw_affiliation_string":"University of Oregon, Oregon, USA","institution_ids":["https://openalex.org/I181233156"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"252","last_page":"266"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9805999994277954,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9805999994277954,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9750000238418579,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9717000126838684,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8001747131347656},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.7280022501945496},{"id":"https://openalex.org/keywords/fault-tolerance","display_name":"Fault tolerance","score":0.6535266637802124},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5688366293907166},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3651919960975647},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.3246743679046631},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.28127506375312805}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8001747131347656},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.7280022501945496},{"id":"https://openalex.org/C63540848","wikidata":"https://www.wikidata.org/wiki/Q3140932","display_name":"Fault tolerance","level":2,"score":0.6535266637802124},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5688366293907166},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3651919960975647},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3246743679046631},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.28127506375312805},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3710848.3710870","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3710848.3710870","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3710848.3710870","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM SIGPLAN Annual Symposium on Principles and Practice of Parallel Programming","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3710848.3710870","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3710848.3710870","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3710848.3710870","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM SIGPLAN Annual Symposium on Principles and Practice of Parallel Programming","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","score":0.6100000143051147,"id":"https://metadata.un.org/sdg/10"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4408029605.pdf"},"referenced_works_count":30,"referenced_works":["https://openalex.org/W1553890549","https://openalex.org/W1995746640","https://openalex.org/W2083613288","https://openalex.org/W2151984682","https://openalex.org/W2268994915","https://openalex.org/W2318507312","https://openalex.org/W2412349256","https://openalex.org/W2485331474","https://openalex.org/W2515526980","https://openalex.org/W2529578574","https://openalex.org/W2580243656","https://openalex.org/W2767260595","https://openalex.org/W2767694495","https://openalex.org/W2883034956","https://openalex.org/W2924040443","https://openalex.org/W2949800959","https://openalex.org/W2985374077","https://openalex.org/W3046764219","https://openalex.org/W3120931476","https://openalex.org/W3129831491","https://openalex.org/W3173669904","https://openalex.org/W3204091362","https://openalex.org/W4233447799","https://openalex.org/W4235427396","https://openalex.org/W4252174618","https://openalex.org/W4287322404","https://openalex.org/W4315588752","https://openalex.org/W4380881139","https://openalex.org/W4388212323","https://openalex.org/W6948096699"],"related_works":["https://openalex.org/W230091440","https://openalex.org/W2233261550","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W2997094352","https://openalex.org/W3216976533","https://openalex.org/W100620283","https://openalex.org/W2495260952","https://openalex.org/W4366179611","https://openalex.org/W2996078371"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4],"demonstrated":[5],"remarkable":[6],"performance":[7,130],"in":[8,28,59,81,116],"various":[9],"natural":[10],"language":[11],"processing":[12],"tasks.":[13],"However,":[14],"the":[15,29,45,60,70,82,88,96,104,113,172],"training":[16,162],"of":[17,37,47,73,98,126],"these":[18,74,99],"models":[19],"is":[20,33,119],"computationally":[21],"intensive":[22],"and":[23,56,84,128,138,165],"susceptible":[24],"to":[25,90,132,134,182],"faults,":[26,100],"particularly":[27],"attention":[30,114],"mechanism,":[31],"which":[32,76],"a":[34],"critical":[35],"component":[36],"transformer-based":[38],"LLMs.":[39,117],"In":[40],"this":[41],"paper,":[42],"we":[43,101],"investigate":[44],"impact":[46,97],"faults":[48],"on":[49,53,122,150,157,161],"LLM":[50,127,147],"training,":[51,86],"focusing":[52],"INF,":[54],"NaN,":[55],"near-INF":[57],"values":[58],"computation":[61],"results":[62],"with":[63,171],"systematic":[64],"fault":[65,123],"injection":[66],"experiments.":[67],"We":[68],"observe":[69],"propagation":[71,124],"patterns":[72,125],"errors,":[75],"can":[77],"trigger":[78],"non-trainable":[79],"states":[80],"model":[83,139],"disrupt":[85],"forcing":[87],"procedure":[89],"load":[91],"from":[92],"checkpoints.":[93],"To":[94],"mitigate":[95],"propose":[102],"ATTNChecker,":[103],"first":[105],"Algorithm-Based":[106],"Fault":[107],"Tolerance":[108],"(ABFT)":[109],"technique":[110],"tailored":[111],"for":[112,145],"mechanism":[115],"ATTNChecker":[118,155,176],"designed":[120],"based":[121],"incorporates":[129],"optimization":[131],"adapt":[133],"both":[135],"system":[136],"reliability":[137],"vulnerability":[140],"while":[141,163],"providing":[142],"lightweight":[143],"protection":[144],"fast":[146],"training.":[148],"Evaluations":[149],"four":[151],"LLMs":[152],"show":[153],"that":[154],"incurs":[156],"average":[158],"7%":[159],"overhead":[160,179],"detecting":[164],"correcting":[166],"all":[167],"extreme":[168],"errors.":[169],"Compared":[170],"state-of-the-art":[173],"checkpoint/restore":[174],"approach,":[175],"reduces":[177],"recovery":[178],"by":[180],"up":[181],"49\u00d7.":[183]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":4}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}