{"id":"https://openalex.org/W4416181688","doi":"https://doi.org/10.1145/3712285.3759864","title":"MLP-Offload: Multi-Level, Multi-Path Offloading for LLM Pre-training to Break the GPU Memory Wall","display_name":"MLP-Offload: Multi-Level, Multi-Path Offloading for LLM Pre-training to Break the GPU Memory Wall","publication_year":2025,"publication_date":"2025-11-12","ids":{"openalex":"https://openalex.org/W4416181688","doi":"https://doi.org/10.1145/3712285.3759864"},"language":"en","primary_location":{"id":"doi:10.1145/3712285.3759864","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3712285.3759864","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3712285.3759864","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049218913","display_name":"Avinash Maurya","orcid":"https://orcid.org/0000-0002-8200-0148"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Avinash Kumar Maurya","raw_affiliation_strings":["Argonne National Laboratory, Lemont, Illinois, USA","ANL - Argonne National Laboratory [Lemont] (9700 S Cass Ave B109, Lemont, IL, 60439 - United States)"],"raw_orcid":"https://orcid.org/0000-0002-8200-0148","affiliations":[{"raw_affiliation_string":"Argonne National Laboratory, Lemont, Illinois, USA","institution_ids":["https://openalex.org/I1282105669"]},{"raw_affiliation_string":"ANL - Argonne National Laboratory [Lemont] (9700 S Cass Ave B109, Lemont, IL, 60439 - United States)","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054011637","display_name":"M. Mustafa Rafique","orcid":"https://orcid.org/0000-0002-5034-2880"},"institutions":[{"id":"https://openalex.org/I155173764","display_name":"Rochester Institute of Technology","ror":"https://ror.org/00v4yb702","country_code":"US","type":"education","lineage":["https://openalex.org/I155173764"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"M. Mustafa Rafique","raw_affiliation_strings":["Rochester Institute of Technology, Rochester, New York, USA","RIT - Rochester Institute of Technology (Rochester, NY 14623 - United States)"],"raw_orcid":"https://orcid.org/0000-0002-5034-2880","affiliations":[{"raw_affiliation_string":"Rochester Institute of Technology, Rochester, New York, USA","institution_ids":["https://openalex.org/I155173764"]},{"raw_affiliation_string":"RIT - Rochester Institute of Technology (Rochester, NY 14623 - United States)","institution_ids":["https://openalex.org/I155173764"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046613458","display_name":"Franck Cappello","orcid":"https://orcid.org/0000-0002-7890-3934"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Franck Cappello","raw_affiliation_strings":["Argonne National Laboratory, Lemont, Illinois, USA","ANL - Argonne National Laboratory [Lemont] (9700 S Cass Ave B109, Lemont, IL, 60439 - United States)"],"raw_orcid":"https://orcid.org/0000-0002-7890-3934","affiliations":[{"raw_affiliation_string":"Argonne National Laboratory, Lemont, Illinois, USA","institution_ids":["https://openalex.org/I1282105669"]},{"raw_affiliation_string":"ANL - Argonne National Laboratory [Lemont] (9700 S Cass Ave B109, Lemont, IL, 60439 - United States)","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5085745891","display_name":"Bogdan Nicolae","orcid":"https://orcid.org/0000-0002-0661-7509"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bogdan Nicolae","raw_affiliation_strings":["Argonne National Laboratory, Lemont, Illinois, USA","ANL - Argonne National Laboratory [Lemont] (9700 S Cass Ave B109, Lemont, IL, 60439 - United States)"],"raw_orcid":"https://orcid.org/0000-0002-0661-7509","affiliations":[{"raw_affiliation_string":"Argonne National Laboratory, Lemont, Illinois, USA","institution_ids":["https://openalex.org/I1282105669"]},{"raw_affiliation_string":"ANL - Argonne National Laboratory [Lemont] (9700 S Cass Ave B109, Lemont, IL, 60439 - United States)","institution_ids":["https://openalex.org/I1282105669"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.40129991,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1381","last_page":"1394"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.941100001335144,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.941100001335144,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.021199999377131462,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.01889999955892563,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/asynchronous-communication","display_name":"Asynchronous communication","score":0.560699999332428},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5555999875068665},{"id":"https://openalex.org/keywords/bandwidth","display_name":"Bandwidth (computing)","score":0.48579999804496765},{"id":"https://openalex.org/keywords/memory-bandwidth","display_name":"Memory bandwidth","score":0.4043000042438507},{"id":"https://openalex.org/keywords/host","display_name":"Host (biology)","score":0.37709999084472656},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.3756999969482422},{"id":"https://openalex.org/keywords/path","display_name":"Path (computing)","score":0.36239999532699585},{"id":"https://openalex.org/keywords/mobile-device","display_name":"Mobile device","score":0.34630000591278076}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8288999795913696},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.560699999332428},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5555999875068665},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.48579999804496765},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.4648999869823456},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.4050000011920929},{"id":"https://openalex.org/C188045654","wikidata":"https://www.wikidata.org/wiki/Q17148339","display_name":"Memory bandwidth","level":2,"score":0.4043000042438507},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.39010000228881836},{"id":"https://openalex.org/C126831891","wikidata":"https://www.wikidata.org/wiki/Q221673","display_name":"Host (biology)","level":2,"score":0.37709999084472656},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.3756999969482422},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.36239999532699585},{"id":"https://openalex.org/C186967261","wikidata":"https://www.wikidata.org/wiki/Q5082128","display_name":"Mobile device","level":2,"score":0.34630000591278076},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.31520000100135803},{"id":"https://openalex.org/C176649486","wikidata":"https://www.wikidata.org/wiki/Q2308807","display_name":"Memory management","level":3,"score":0.3131999969482422},{"id":"https://openalex.org/C2778710394","wikidata":"https://www.wikidata.org/wiki/Q4179813","display_name":"Live migration","level":4,"score":0.303600013256073},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.30309998989105225},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.28929999470710754},{"id":"https://openalex.org/C47487241","wikidata":"https://www.wikidata.org/wiki/Q5227230","display_name":"Data access","level":2,"score":0.2824000120162964},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.2678999900817871},{"id":"https://openalex.org/C64142963","wikidata":"https://www.wikidata.org/wiki/Q1153902","display_name":"Phase-change memory","level":3,"score":0.26649999618530273},{"id":"https://openalex.org/C2989134064","wikidata":"https://www.wikidata.org/wiki/Q288510","display_name":"Execution time","level":2,"score":0.2572999894618988},{"id":"https://openalex.org/C2984118289","wikidata":"https://www.wikidata.org/wiki/Q29954","display_name":"Power consumption","level":3,"score":0.25060001015663147},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.25029999017715454}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3712285.3759864","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3712285.3759864","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2509.02480","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.02480","pdf_url":"https://arxiv.org/pdf/2509.02480","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:HAL:hal-05269984v1","is_oa":true,"landing_page_url":"https://inria.hal.science/hal-05269984","pdf_url":"https://inria.hal.science/hal-05269984v1/document","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"SC'25: 38th International Conference for High Performance Computing, Networking, Storage and Analytics, Nov 2025, St Louis, United States. ⟨10.1145/3712285.3759864⟩","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.1145/3712285.3759864","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3712285.3759864","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G102163711","display_name":null,"funder_award_id":"DE\u2010 AC02\u201006CH11357","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G107546177","display_name":null,"funder_award_id":"DE- AC02-06-CH11357","funder_id":"https://openalex.org/F4320338284","funder_display_name":"Argonne National Laboratory"},{"id":"https://openalex.org/G2131453672","display_name":"Collaborative Research: CNS Core: Medium:HardLambda: A new FaaS Abstraction for Cross-Stack Resource Management in Disaggregated Datacenters","funder_award_id":"2106635","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3075337988","display_name":null,"funder_award_id":"06CH11357","funder_id":"https://openalex.org/F4320338284","funder_display_name":"Argonne National Laboratory"},{"id":"https://openalex.org/G3357838806","display_name":"Collaborative Research: Elements: VLCC-States: Versioned Lineage-Driven Checkpointing of Composable States","funder_award_id":"2411386","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3714508591","display_name":null,"funder_award_id":"DEAC02-06CH11357\\/0F-60169","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G4552540348","display_name":null,"funder_award_id":"0F-60169","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G5367729914","display_name":null,"funder_award_id":"0F-60169","funder_id":"https://openalex.org/F4320338284","funder_display_name":"Argonne National Laboratory"},{"id":"https://openalex.org/G6189456144","display_name":null,"funder_award_id":"2411386, 2411387, 2106635","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6848031779","display_name":null,"funder_award_id":"06CH11357","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G878601127","display_name":null,"funder_award_id":"DE-AC02-06CH11357","funder_id":"https://openalex.org/F4320337506","funder_display_name":"Advanced Scientific Computing Research"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306084","display_name":"U.S. Department of Energy","ror":"https://ror.org/01bj3aw27"},{"id":"https://openalex.org/F4320337506","display_name":"Advanced Scientific Computing Research","ror":"https://ror.org/0012c7r22"},{"id":"https://openalex.org/F4320338284","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W3019389960","https://openalex.org/W3081168214","https://openalex.org/W3129831491","https://openalex.org/W3205803342","https://openalex.org/W4281926585","https://openalex.org/W4288083766","https://openalex.org/W4386709668","https://openalex.org/W4386768656","https://openalex.org/W4387302750","https://openalex.org/W4393115925","https://openalex.org/W4393145114","https://openalex.org/W4393407018","https://openalex.org/W4399427731","https://openalex.org/W4399757647","https://openalex.org/W4399794472","https://openalex.org/W4400125901","https://openalex.org/W4401486467","https://openalex.org/W4404340628","https://openalex.org/W4406265565"],"related_works":[],"abstract_inverted_index":{"Training":[0],"LLMs":[1],"larger":[2],"than":[3],"the":[4,15,56,97,106,109,115,142,159,181],"aggregated":[5],"memory":[6,30],"of":[7,18,39,59,99,114],"multiple":[8,146],"GPUs":[9],"is":[10],"increasingly":[11],"necessary":[12],"due":[13,124],"to":[14,22,125,140,154,168,180],"faster":[16,177],"growth":[17],"LLM":[19,81,183],"sizes":[20],"compared":[21,179],"GPU":[23],"memory.":[24],"To":[25,65],"this":[26,66],"end,":[27,67],"multi-tier":[28,44],"host":[29],"or":[31],"disk":[32],"offloading":[33,48,75,127],"techniques":[34],"are":[35],"proposed":[36],"by":[37,86,132],"state":[38],"art.":[40],"Despite":[41],"advanced":[42],"asynchronous":[43],"read/write":[45],"strategies,":[46],"such":[47,101],"strategies":[49],"result":[50],"in":[51,55,62,148],"significant":[52],"I/O":[53,88,103,112,129,156],"overheads":[54,104],"critical":[57],"path":[58],"training,":[60],"resulting":[61],"slower":[63],"iterations.":[64],"we":[68,135],"propose":[69],"MLP-Offload,":[70,100],"a":[71,149],"novel":[72],"multi-level,":[73],"multi-path":[74],"engine":[76],"specifically":[77],"designed":[78],"for":[79],"optimizing":[80],"training":[82,184],"on":[83,165],"resource-constrained":[84],"setups":[85],"mitigating":[87],"bottlenecks.":[89,130],"We":[90],"make":[91],"several":[92],"key":[93],"observations":[94],"that":[95,172],"drive":[96],"design":[98,136],"as":[102],"during":[105,158],"update":[107,162],"dominate":[108],"iteration":[110],"time;":[111],"bandwidth":[113],"third-level":[116],"remote":[117],"storage":[118],"tier":[119],"remains":[120],"unutilized;":[121],"and,":[122],"contention":[123],"concurrent":[126],"amplifies":[128],"Driven":[131],"these":[133],"insights,":[134],"and":[137,151,161],"implement":[138],"MLP-Offload":[139,173],"offload":[141],"optimizer":[143],"states":[144],"across":[145],"tiers":[147],"cache-efficient":[150],"concurrency-controlled":[152],"fashion":[153],"mitigate":[155],"bottlenecks":[157],"backward":[160],"phases.":[163],"Evaluations":[164],"models":[166],"up":[167],"280B":[169],"parameters":[170],"shows":[171],"achieves":[174],"2.5":[175],"\u00d7":[176],"iterations":[178],"state-of-the-art":[182],"runtimes.":[185]},"counts_by_year":[],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2025-10-10T00:00:00"}