{"id":"https://openalex.org/W4411639943","doi":"https://doi.org/10.1109/tpds.2025.3583165","title":"Oases: Efficient Large-Scale Model Training on Commodity Servers via Overlapped and Automated Tensor Model Parallelism","display_name":"Oases: Efficient Large-Scale Model Training on Commodity Servers via Overlapped and Automated Tensor Model Parallelism","publication_year":2025,"publication_date":"2025-06-25","ids":{"openalex":"https://openalex.org/W4411639943","doi":"https://doi.org/10.1109/tpds.2025.3583165"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2025.3583165","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2025.3583165","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Shengwei Li","orcid":"https://orcid.org/0000-0002-7419-1511"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shengwei Li","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, Hunan, China","National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology in Changsha, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0002-7419-1511","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology in Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055426388","display_name":"Zhiquan Lai","orcid":"https://orcid.org/0000-0002-3458-4732"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiquan Lai","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, Hunan, China","National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology in Changsha, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0002-3458-4732","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology in Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Dongsheng Li","orcid":"https://orcid.org/0000-0001-9743-2034"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongsheng Li","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, Hunan, China","National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology in Changsha, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0001-9743-2034","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology in Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101206777","display_name":"Yanqi Hao","orcid":null},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanqi Hao","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, Hunan, China","National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology in Changsha, Hunan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology in Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Weijie Liu","orcid":"https://orcid.org/0000-0002-0354-9467"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weijie Liu","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, Hunan, China","National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology in Changsha, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0002-0354-9467","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology in Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074708780","display_name":"Keshi Ge","orcid":"https://orcid.org/0000-0002-0669-6892"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Keshi Ge","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, Hunan, China","National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology in Changsha, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0002-0669-6892","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology in Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001375179","display_name":"Xiaoge Deng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiaoge Deng","raw_affiliation_strings":["Intelligent Game and Decision Lab, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-0622-1202","affiliations":[{"raw_affiliation_string":"Intelligent Game and Decision Lab, Beijing, China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100625354","display_name":"Kai L\u00fc","orcid":"https://orcid.org/0000-0002-6378-7002"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kai Lu","raw_affiliation_strings":["National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, Hunan, China","National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology in Changsha, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0002-6378-7002","affiliations":[{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology in Changsha, Hunan, China","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.7655,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.71462574,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":"36","issue":"9","first_page":"1828","last_page":"1840"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9524999856948853,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9524999856948853,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8043094277381897},{"id":"https://openalex.org/keywords/server","display_name":"Server","score":0.7207235097885132},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.6473045349121094},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5724893808364868},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5628923177719116},{"id":"https://openalex.org/keywords/tensor","display_name":"Tensor (intrinsic definition)","score":0.4733595550060272},{"id":"https://openalex.org/keywords/commodity","display_name":"Commodity","score":0.4403783082962036},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.42777952551841736},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.22262093424797058}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8043094277381897},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.7207235097885132},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.6473045349121094},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5724893808364868},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5628923177719116},{"id":"https://openalex.org/C155281189","wikidata":"https://www.wikidata.org/wiki/Q3518150","display_name":"Tensor (intrinsic definition)","level":2,"score":0.4733595550060272},{"id":"https://openalex.org/C2779439359","wikidata":"https://www.wikidata.org/wiki/Q317088","display_name":"Commodity","level":2,"score":0.4403783082962036},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.42777952551841736},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.22262093424797058},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C34447519","wikidata":"https://www.wikidata.org/wiki/Q179522","display_name":"Market economy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2025.3583165","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2025.3583165","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2277931485","display_name":null,"funder_award_id":"62421002","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3606055544","display_name":null,"funder_award_id":"62025208","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":32,"referenced_works":["https://openalex.org/W2056999868","https://openalex.org/W2057332538","https://openalex.org/W2131613942","https://openalex.org/W2489773810","https://openalex.org/W2926767350","https://openalex.org/W2963786636","https://openalex.org/W2969388332","https://openalex.org/W2975712713","https://openalex.org/W2991040477","https://openalex.org/W3086105743","https://openalex.org/W3129831491","https://openalex.org/W3132107458","https://openalex.org/W3193985311","https://openalex.org/W3202134982","https://openalex.org/W3207427731","https://openalex.org/W4220703317","https://openalex.org/W4249982028","https://openalex.org/W4285173593","https://openalex.org/W4285225959","https://openalex.org/W4289646389","https://openalex.org/W4294433898","https://openalex.org/W4310282800","https://openalex.org/W4312060029","https://openalex.org/W4312681979","https://openalex.org/W4318541673","https://openalex.org/W4321636583","https://openalex.org/W4327694855","https://openalex.org/W4360831999","https://openalex.org/W4382119135","https://openalex.org/W4385245566","https://openalex.org/W4386709668","https://openalex.org/W4386768656"],"related_works":["https://openalex.org/W2092530219","https://openalex.org/W2388464034","https://openalex.org/W2533125852","https://openalex.org/W2140460949","https://openalex.org/W2105580438","https://openalex.org/W2057435755","https://openalex.org/W2045183646","https://openalex.org/W2162409446","https://openalex.org/W1595672120","https://openalex.org/W4230999561"],"abstract_inverted_index":{"Deep":[0],"learning":[1],"is":[2,12,56,140],"experiencing":[3],"a":[4,99],"rise":[5],"in":[6],"large-scale":[7,10,18,91],"models.":[8],"Training":[9],"models":[11,19],"costly,":[13],"prompting":[14],"researchers":[15,25],"to":[16,89,104,131,142,164,186],"train":[17],"on":[20,44,94,153],"commodity":[21,95,159],"servers":[22],"that":[23,110,120,171],"more":[24],"can":[26],"access.":[27],"The":[28],"massive":[29],"number":[30],"of":[31,36,129,146,175,184],"parameters":[32],"necessitates":[33],"the":[34,51,59,73,117,123,144,179],"use":[35],"model":[37,48,53,60,92,125,143,155],"parallelism":[38,54],"training":[39,45,74,93,101],"methods.":[40],"Existing":[41],"studies":[42],"focus":[43],"with":[46,86],"pipeline":[47],"parallelism.":[49],"However,":[50],"tensor":[52],"(TMP)":[55],"inevitable":[57],"when":[58],"size":[61],"keeps":[62],"increasing,":[63],"where":[64],"frequent":[65],"data-dependent":[66],"communication":[67,88,107],"and":[68,108,157,161,182],"computation":[69,109],"operations":[70],"significantly":[71],"reduce":[72],"efficiency.":[75],"In":[76],"this":[77],"paper,":[78],"we":[79,115],"present":[80],"Oases,":[81],"an":[82],"automated":[83],"TMP":[84,130],"method":[85],"overlapped":[87,147],"accelerate":[90],"servers.":[96],"Oases":[97,118,138,152,163,172],"proposes":[98],"fine-grained":[100],"operation":[102],"schedule":[103],"maximize":[105],"overlapping":[106],"have":[111],"data":[112],"dependence.":[113],"Additionally,":[114],"design":[116],"planner":[119,139],"searches":[121],"for":[122],"best":[124],"parameter":[126],"partition":[127],"strategy":[128],"achieve":[132],"further":[133],"accelerations.":[134],"Unlike":[135],"existing":[136],"methods,":[137],"tailored":[141],"cost":[145],"communication-computation":[148],"operations.":[149],"We":[150],"evaluate":[151],"various":[154],"settings":[156],"two":[158],"clusters,":[160],"compare":[162],"four":[165],"state-of-the-art":[166],"implementations.":[167],"Experimental":[168],"results":[169],"show":[170],"achieves":[173],"speedups":[174,183],"1.01\u20131.48":[176],"\u00d7":[177,188],"over":[178,189],"fastest":[180],"baseline,":[181],"up":[185],"1.95":[187],"Megatron.":[190]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-02-03T00:53:05.648605","created_date":"2025-10-10T00:00:00"}