{"id":"https://openalex.org/W4386323709","doi":"https://doi.org/10.1109/jcc59055.2023.00006","title":"OLM<sup>2</sup>: Automatic Optimal Strategy Generating for Large-Scale Model Training with Limited-Memory","display_name":"OLM<sup>2</sup>: Automatic Optimal Strategy Generating for Large-Scale Model Training with Limited-Memory","publication_year":2023,"publication_date":"2023-07-01","ids":{"openalex":"https://openalex.org/W4386323709","doi":"https://doi.org/10.1109/jcc59055.2023.00006"},"language":"en","primary_location":{"id":"doi:10.1109/jcc59055.2023.00006","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/jcc59055.2023.00006","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE International Conference on Joint Cloud Computing (JCC)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5082769820","display_name":"Zhilin Yang","orcid":null},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhilin Yang","raw_affiliation_strings":["National University of Defense Technology,College of Computer Science,National Key Laboratory of Parallel and Distributed Computing,Changsha 410073,China","National Key Laboratory of Parallel and Distributed Computing, College of Computer Science, National University of Defense Technology, Changsha 410073, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Defense Technology,College of Computer Science,National Key Laboratory of Parallel and Distributed Computing,Changsha 410073,China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science, National University of Defense Technology, Changsha 410073, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102970145","display_name":"Yu Tang","orcid":"https://orcid.org/0000-0002-8595-1547"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu Tang","raw_affiliation_strings":["National University of Defense Technology,College of Computer Science,National Key Laboratory of Parallel and Distributed Computing,Changsha 410073,China","National Key Laboratory of Parallel and Distributed Computing, College of Computer Science, National University of Defense Technology, Changsha 410073, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Defense Technology,College of Computer Science,National Key Laboratory of Parallel and Distributed Computing,Changsha 410073,China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science, National University of Defense Technology, Changsha 410073, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043045076","display_name":"Linbo Qiao","orcid":"https://orcid.org/0000-0002-8285-2738"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Linbo Qiao","raw_affiliation_strings":["National University of Defense Technology,College of Computer Science,National Key Laboratory of Parallel and Distributed Computing,Changsha 410073,China","National Key Laboratory of Parallel and Distributed Computing, College of Computer Science, National University of Defense Technology, Changsha 410073, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Defense Technology,College of Computer Science,National Key Laboratory of Parallel and Distributed Computing,Changsha 410073,China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science, National University of Defense Technology, Changsha 410073, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100355610","display_name":"Xi Yang","orcid":"https://orcid.org/0009-0006-8001-8092"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xi Yang","raw_affiliation_strings":["National University of Defense Technology,College of Computer Science,National Key Laboratory of Parallel and Distributed Computing,Changsha 410073,China","National Key Laboratory of Parallel and Distributed Computing, College of Computer Science, National University of Defense Technology, Changsha 410073, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Defense Technology,College of Computer Science,National Key Laboratory of Parallel and Distributed Computing,Changsha 410073,China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science, National University of Defense Technology, Changsha 410073, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5011008665","display_name":"Zhen Huang","orcid":"https://orcid.org/0000-0003-4819-373X"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhen Huang","raw_affiliation_strings":["National University of Defense Technology,College of Computer Science,National Key Laboratory of Parallel and Distributed Computing,Changsha 410073,China","National Key Laboratory of Parallel and Distributed Computing, College of Computer Science, National University of Defense Technology, Changsha 410073, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Defense Technology,College of Computer Science,National Key Laboratory of Parallel and Distributed Computing,Changsha 410073,China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science, National University of Defense Technology, Changsha 410073, China","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I170215575"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"13","issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9934999942779541,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9929999709129333,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6800207495689392},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.6753992438316345},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5484429597854614},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.45962846279144287},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4214441776275635},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.41912221908569336}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6800207495689392},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6753992438316345},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5484429597854614},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.45962846279144287},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4214441776275635},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.41912221908569336},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/jcc59055.2023.00006","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/jcc59055.2023.00006","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE International Conference on Joint Cloud Computing (JCC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W2112796928","https://openalex.org/W2140679639","https://openalex.org/W2194775991","https://openalex.org/W2338908902","https://openalex.org/W2489529491","https://openalex.org/W2618530766","https://openalex.org/W2954698599","https://openalex.org/W2963163009","https://openalex.org/W2971843695","https://openalex.org/W3012479151","https://openalex.org/W3012514909","https://openalex.org/W3134582802","https://openalex.org/W3157864729","https://openalex.org/W4246193833","https://openalex.org/W4286206342","https://openalex.org/W4287391717","https://openalex.org/W4288089799","https://openalex.org/W4293404878","https://openalex.org/W4301361180","https://openalex.org/W4327694855","https://openalex.org/W6637373629","https://openalex.org/W6680532216","https://openalex.org/W6703652217","https://openalex.org/W6717768942","https://openalex.org/W6768723914","https://openalex.org/W6769627184","https://openalex.org/W6779999780","https://openalex.org/W6788811087","https://openalex.org/W6791276965"],"related_works":["https://openalex.org/W230091440","https://openalex.org/W2233261550","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W2997094352","https://openalex.org/W3216976533","https://openalex.org/W100620283","https://openalex.org/W2495260952","https://openalex.org/W4394050964","https://openalex.org/W2551249631"],"abstract_inverted_index":{"The":[0],"scale":[1],"of":[2,8,23,53,74,139],"model":[3,24,91],"parameters":[4],"and":[5,27,40,69,85,106,142],"the":[6,20,48,65,72,75,109,121,130,137,151,169,177,182],"amount":[7],"training":[9,170,179],"data":[10],"is":[11,87,113],"exponentially":[12],"increasing.":[13],"It":[14],"requires":[15],"more":[16],"GPU":[17],"memory":[18,32,110,122,132,153,166],"with":[19,82],"exponential":[21],"increasement":[22],"parameters.":[25],"Recomputation":[26],"swapping":[28,86,111],"are":[29,42,55],"two":[30,49],"main":[31],"optimization":[33,44,117,133],"methods":[34],"that":[35,46,158],"have":[36],"been":[37],"extensively":[38],"studied,":[39],"there":[41],"also":[43],"strategies":[45],"combine":[47],"methods.":[50],"However,":[51],"most":[52],"them":[54],"based":[56],"on":[57],"heuristic":[58],"search":[59,80],"strategies,":[60],"which":[61,119],"do":[62],"not":[63],"explore":[64],"complete":[66],"solution":[67,76],"space":[68],"can\u2019t":[70],"guarantee":[71],"optimality":[73],"results.":[77],"An":[78],"optimal":[79,99,131],"strategy":[81,100,112],"tensor-level":[83],"recomputation":[84,105,141],"expected":[88],"in":[89,165,194],"large-scale":[90],"training.":[92],"In":[93],"this":[94,144],"paper,":[95],"we":[96],"propose":[97],"an":[98,116],"searching":[101],"algorithm":[102],"combining":[103],"tensor-based":[104],"swapping.":[107],"Specifically,":[108],"reformulated":[114],"as":[115],"problem,":[118],"converts":[120],"constraints":[123],"into":[124],"mixed":[125],"integer":[126],"programming,":[127],"to":[128,186],"find":[129],"strategy.":[134],"By":[135],"leveraging":[136],"advantages":[138],"both":[140],"swapping,":[143],"approach":[145,189],"minimizes":[146],"computation":[147,195],"consumption":[148],"without":[149],"exceeding":[150],"available":[152],"limitation.":[154],"Experimental":[155],"results":[156],"show":[157],"our":[159,173,188],"method":[160,174],"exhibits":[161],"about":[162,191],"60%":[163],"reduction":[164,193],"requirements":[167],"during":[168],"process.":[171],"Furthermore,":[172],"can":[175],"reduce":[176],"overall":[178],"time":[180],"beyond":[181],"existing":[183],"algorithms.":[184],"Compared":[185],"Checkmate,":[187],"achieves":[190],"0.3\u20130.9%":[192],"cost":[196],"per":[197],"iteration.":[198]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}