{"id":"https://openalex.org/W7161564317","doi":"https://doi.org/10.1109/tpds.2026.3694405","title":"<i>DataStates-LLM</i> : Scalable Checkpointing for Transformer Models Using Composable State Providers","display_name":"<i>DataStates-LLM</i> : Scalable Checkpointing for Transformer Models Using Composable State Providers","publication_year":2026,"publication_date":"2026-05-18","ids":{"openalex":"https://openalex.org/W7161564317","doi":"https://doi.org/10.1109/tpds.2026.3694405"},"language":null,"primary_location":{"id":"doi:10.1109/tpds.2026.3694405","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2026.3694405","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136019497","display_name":"Avinash Maurya","orcid":null},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Avinash Maurya","raw_affiliation_strings":["Argonne National Laboratory, Lemont, IL, USA"],"raw_orcid":"https://orcid.org/0000-0002-8200-0148","affiliations":[{"raw_affiliation_string":"Argonne National Laboratory, Lemont, IL, USA","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135727934","display_name":"M. Mustafa Rafique","orcid":"https://orcid.org/0000-0002-5034-2880"},"institutions":[{"id":"https://openalex.org/I155173764","display_name":"Rochester Institute of Technology","ror":"https://ror.org/00v4yb702","country_code":"US","type":"education","lineage":["https://openalex.org/I155173764"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"M. Mustafa Rafique","raw_affiliation_strings":["Rochester Institute of Technology, Rochester, NY, USA"],"raw_orcid":"https://orcid.org/0000-0002-5034-2880","affiliations":[{"raw_affiliation_string":"Rochester Institute of Technology, Rochester, NY, USA","institution_ids":["https://openalex.org/I155173764"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046613458","display_name":"Franck Cappello","orcid":"https://orcid.org/0000-0002-7890-3934"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Franck Cappello","raw_affiliation_strings":["Argonne National Laboratory, Lemont, IL, USA"],"raw_orcid":"https://orcid.org/0000-0002-7890-3934","affiliations":[{"raw_affiliation_string":"Argonne National Laboratory, Lemont, IL, USA","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5136366884","display_name":"Bogdan Nicolae","orcid":null},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bogdan Nicolae","raw_affiliation_strings":["Argonne National Laboratory, Lemont, IL, USA"],"raw_orcid":"https://orcid.org/0000-0002-0661-7509","affiliations":[{"raw_affiliation_string":"Argonne National Laboratory, Lemont, IL, USA","institution_ids":["https://openalex.org/I1282105669"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.62324286,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"37","issue":"9","first_page":"2037","last_page":"2049"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10772","display_name":"Distributed systems and fault tolerance","score":0.08410000056028366,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10772","display_name":"Distributed systems and fault tolerance","score":0.08410000056028366,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.07779999822378159,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10363","display_name":"Low-power high-performance VLSI design","score":0.035599999129772186,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6672000288963318},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.454800009727478},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.4117000102996826},{"id":"https://openalex.org/keywords/server","display_name":"Server","score":0.35280001163482666},{"id":"https://openalex.org/keywords/fault-tolerance","display_name":"Fault tolerance","score":0.3260999917984009},{"id":"https://openalex.org/keywords/processor-scheduling","display_name":"Processor scheduling","score":0.322299987077713}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8543000221252441},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6672000288963318},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5148000121116638},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.454800009727478},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.4447999894618988},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4117000102996826},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.36410000920295715},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.35280001163482666},{"id":"https://openalex.org/C63540848","wikidata":"https://www.wikidata.org/wiki/Q3140932","display_name":"Fault tolerance","level":2,"score":0.3260999917984009},{"id":"https://openalex.org/C2984822820","wikidata":"https://www.wikidata.org/wiki/Q1123036","display_name":"Processor scheduling","level":3,"score":0.322299987077713},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.29789999127388},{"id":"https://openalex.org/C3020028006","wikidata":"https://www.wikidata.org/wiki/Q9158","display_name":"Electronic mail","level":2,"score":0.27709999680519104},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.258899986743927}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2026.3694405","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2026.3694405","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306084","display_name":"U.S. Department of Energy","ror":"https://ror.org/01bj3aw27"},{"id":"https://openalex.org/F4320332359","display_name":"Office of Science","ror":"https://ror.org/00mmn6b08"},{"id":"https://openalex.org/F4320337506","display_name":"Advanced Scientific Computing Research","ror":"https://ror.org/0012c7r22"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"rapid":[1],"growth":[2],"of":[3,16,23,47,78,90,152,181],"Large":[4,8],"Transformer-based":[5],"models,":[6],"specifically":[7],"Language":[9],"Models":[10],"(LLMs),":[11],"now":[12],"scaling":[13],"to":[14,116,141,161,193,207,219,222],"trillions":[15],"parameters,":[17],"has":[18],"necessitated":[19],"training":[20,56,215],"across":[21,96],"thousands":[22],"GPUs":[24],"using":[25],"complex":[26],"hybrid":[27],"parallelism":[28],"strategies":[29],"(e.g.,":[30],"data,":[31],"tensor,":[32],"and":[33,58,94,105,122,158,177,212,229],"pipeline":[34],"parallelism).":[35],"Checkpointing":[36],"this":[37,127],"massive,":[38],"distributed":[39],"state":[40,69,143],"is":[41],"critical":[42],"for":[43],"a":[44,132],"wide":[45],"range":[46],"use":[48],"cases,":[49],"such":[50],"as":[51,70],"resilience,":[52],"suspend-resume,":[53],"investigating":[54],"undesirable":[55],"trajectories,":[57],"explaining":[59],"model":[60,68,153],"evolution.":[61],"However,":[62],"existing":[63],"checkpointing":[64,210],"solutions":[65],"typically":[66],"treat":[67],"opaque":[71],"binary":[72],"blobs,":[73],"ignoring":[74],"the":[75,79,150,156,179,227],"\u201c3D":[76],"heterogeneity\u201d":[77],"underlying":[80],"data":[81,99,146],"structures\u2013varying":[82],"by":[83,217],"memory":[84],"location":[85],"(GPU":[86],"vs.":[87,102],"Host),":[88],"number":[89],"\u201clogical\u201d":[91],"objects":[92],"sharded":[93],"split":[95],"multiple":[97],"files,":[98],"types":[100],"(tensors":[101],"Python":[103],"objects),":[104],"their":[106],"serialization":[107,180,228],"requirements.":[108],"This":[109],"results":[110,201],"in":[111,232],"significant":[112],"runtime":[113],"overheads":[114],"due":[115],"blocking":[117],"device-to-host":[118],"transfers,":[119],"data-oblivious":[120],"serialization,":[121],"storage":[123],"I/O":[124],"contention.":[125],"In":[126],"paper,":[128],"we":[129,171],"introduce":[130],"DataStates-LLM,":[131],"novel":[133],"check":[134],"pointing":[135],"architecture":[136],"that":[137,203],"leverages":[138],"State":[139,169],"Providers":[140],"decouple":[142],"abstraction":[144],"from":[145],"movement.":[147],"DataStates-LLM":[148,189,204],"exploits":[149],"immutability":[151],"parameters":[154,195],"during":[155],"forward":[157],"backward":[159],"passes":[160],"perform":[162],"\u201clazy\u201d,":[163],"non-blocking":[164],"asynchronous":[165],"snapshots.":[166],"By":[167],"introducing":[168],"Providers,":[170],"efficiently":[172],"coalesce":[173],"fragmented,":[174],"heterogeneous":[175],"shards":[176],"overlap":[178],"metadata":[182],"with":[183],"bulk":[184],"tensor":[185],"I/O.":[186],"We":[187],"evaluate":[188],"on":[190,196],"models":[191],"up":[192,206,218],"70B":[194],"256":[197],"A100-40GB":[198],"GPUs.":[199],"Our":[200],"demonstrate":[202],"achieves":[205],"4\u00d7":[208],"higher":[209],"throughput":[211],"reduces":[213],"end-to-end":[214],"time":[216],"2.2\u00d7":[220],"compared":[221],"state-of-the-art":[223],"solutions,":[224],"effectively":[225],"mitigating":[226],"heterogeneity":[230],"bottlenecks":[231],"extreme-scale":[233],"LLM":[234],"training.":[235]},"counts_by_year":[],"updated_date":"2026-07-15T05:50:42.429089","created_date":"2026-05-19T00:00:00"}