{"id":"https://openalex.org/W7116354707","doi":"https://doi.org/10.1145/3754598.3754606","title":"Amber: Towards Fast and Space-Efficient Incremental Checkpointing in Large Language Model Training","display_name":"Amber: Towards Fast and Space-Efficient Incremental Checkpointing in Large Language Model Training","publication_year":2025,"publication_date":"2025-09-08","ids":{"openalex":"https://openalex.org/W7116354707","doi":"https://doi.org/10.1145/3754598.3754606"},"language":null,"primary_location":{"id":"doi:10.1145/3754598.3754606","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3754598.3754606","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 54th International Conference on Parallel Processing","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3754598.3754606","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120871141","display_name":"Zhiqiang Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiqiang Wang","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0009-0005-1710-5014","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Wenzhe Zhu","orcid":"https://orcid.org/0000-0002-3965-2597"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenzhe Zhu","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0000-0002-3965-2597","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101345899","display_name":"Zaigui Zhang","orcid":"https://orcid.org/0009-0009-6075-1221"},"institutions":[{"id":"https://openalex.org/I4210144143","display_name":"Inspur (China)","ror":"https://ror.org/0474p4r72","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210144143"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zaigui Zhang","raw_affiliation_strings":["Jinan Inspur Data Technology Co., Ltd., Jinan, China"],"raw_orcid":"https://orcid.org/0009-0009-6075-1221","affiliations":[{"raw_affiliation_string":"Jinan Inspur Data Technology Co., Ltd., Jinan, China","institution_ids":["https://openalex.org/I4210144143"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120926050","display_name":"Chaomei Yan","orcid":null},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chaomei Yan","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0009-0007-5793-1497","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Fan Guo","orcid":"https://orcid.org/0009-0007-6842-5899"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fan Guo","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0009-0007-6842-5899","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yongkun Li","orcid":"https://orcid.org/0000-0002-3743-8511"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yongkun Li","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0000-0002-3743-8511","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5120873324","display_name":"Yinlong Xu","orcid":null},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]},{"id":"https://openalex.org/I16365422","display_name":"Hefei University of Technology","ror":"https://ror.org/02czkny70","country_code":"CN","type":"education","lineage":["https://openalex.org/I16365422"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yinlong Xu","raw_affiliation_strings":["Anhui Provincial Key Laboratory of High Performance Computing, USTC, Hefei, China and University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0000-0001-9586-0561","affiliations":[{"raw_affiliation_string":"Anhui Provincial Key Laboratory of High Performance Computing, USTC, Hefei, China and University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041","https://openalex.org/I16365422"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"678","last_page":"688"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10772","display_name":"Distributed systems and fault tolerance","score":0.262800008058548,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10772","display_name":"Distributed systems and fault tolerance","score":0.262800008058548,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.09939999878406525,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0868000015616417,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.7429999709129333},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.7225000262260437},{"id":"https://openalex.org/keywords/fault-tolerance","display_name":"Fault tolerance","score":0.6193000078201294},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.46480000019073486},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4575999975204468},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4002000093460083},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.36570000648498535}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.853600025177002},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.7429999709129333},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.7225000262260437},{"id":"https://openalex.org/C63540848","wikidata":"https://www.wikidata.org/wiki/Q3140932","display_name":"Fault tolerance","level":2,"score":0.6193000078201294},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5304999947547913},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.46480000019073486},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4575999975204468},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4002000093460083},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37709999084472656},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.36570000648498535},{"id":"https://openalex.org/C2989134064","wikidata":"https://www.wikidata.org/wiki/Q288510","display_name":"Execution time","level":2,"score":0.33239999413490295},{"id":"https://openalex.org/C175551986","wikidata":"https://www.wikidata.org/wiki/Q47089","display_name":"Fault (geology)","level":2,"score":0.2831999957561493},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2806999981403351},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.27570000290870667},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.27489998936653137},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2736999988555908},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.27230000495910645},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2646999955177307},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.2644999921321869},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.2540999948978424}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3754598.3754606","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3754598.3754606","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 54th International Conference on Parallel Processing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3754598.3754606","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3754598.3754606","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 54th International Conference on Parallel Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.5200691819190979,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":21,"referenced_works":["https://openalex.org/W114517082","https://openalex.org/W1988888548","https://openalex.org/W2606891064","https://openalex.org/W3021124033","https://openalex.org/W3048079516","https://openalex.org/W3081168214","https://openalex.org/W3091097978","https://openalex.org/W3138516171","https://openalex.org/W3204998121","https://openalex.org/W4220741164","https://openalex.org/W4316661233","https://openalex.org/W4317038433","https://openalex.org/W4321446170","https://openalex.org/W4386840193","https://openalex.org/W4387302750","https://openalex.org/W4390097689","https://openalex.org/W4394923484","https://openalex.org/W4398796293","https://openalex.org/W4399757647","https://openalex.org/W4400233507","https://openalex.org/W4404385781"],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"have":[3],"transformed":[4],"numerous":[5],"domains":[6],"with":[7],"their":[8,12,62],"exceptional":[9],"capabilities.":[10],"However,":[11,53],"training":[13,26,95,107],"processes":[14],"are":[15],"inherently":[16],"prone":[17],"to":[18,21,30,61,151,157],"failures":[19],"due":[20],"the":[22,54,88],"massive":[23],"scale":[24],"of":[25,32,34,40,57,65,72,93,164],"clusters,":[27],"involving":[28],"thousands":[29,33],"tens":[31],"GPUs":[35],"and":[36,74,91,114,145],"requiring":[37],"several":[38],"months":[39],"uninterrupted":[41],"computation.":[42],"Checkpointing":[43],"serves":[44],"as":[45],"a":[46,104,122,139,168],"critical":[47],"mechanism":[48],"for":[49],"ensuring":[50],"fault":[51],"tolerance.":[52],"substantial":[55],"size":[56],"LLM":[58,94,106],"checkpoints,":[59],"attributed":[60],"vast":[63],"number":[64],"parameters,":[66],"poses":[67],"significant":[68,127],"challenges":[69],"in":[70,86],"terms":[71],"time":[73],"storage":[75,115,173],"overheads.":[76],"While":[77],"existing":[78],"approaches":[79],"offer":[80],"partial":[81],"solutions,":[82],"they":[83],"remain":[84],"inadequate":[85],"addressing":[87],"unique":[89],"demands":[90],"complexities":[92],"at":[96],"scale.":[97],"To":[98],"this":[99],"end,":[100],"we":[101],"propose":[102],"Amber,":[103],"novel":[105],"framework":[108],"that":[109,124,147],"significantly":[110],"improves":[111],"checkpointing":[112,155,160],"speed":[113],"efficiency":[116],"by":[117],"leveraging":[118],"selective":[119],"incremental":[120],"checkpointing,":[121],"technique":[123],"selectively":[125],"checkpoints":[126],"parameter":[128],"updates":[129],"while":[130,171],"omitting":[131],"minor":[132],"changes.":[133],"We":[134],"implement":[135],"Amber":[136,148],"atop":[137],"PyTorch,":[138],"widely":[140],"adopted":[141],"deep":[142],"learning":[143],"framework,":[144],"demonstrate":[146],"achieves":[149],"up":[150],"10\u2013155":[152],"\u00d7":[153],"faster":[154],"compared":[156],"state-of-the-art":[158],"full":[159],"methods":[161],"across":[162],"models":[163],"varying":[165],"scales":[166],"on":[167],"single":[169],"GPU,":[170],"maintaining":[172],"overhead":[174],"below":[175],"3%.":[176]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-12-21T00:00:00"}