{"id":"https://openalex.org/W4206790491","doi":"https://doi.org/10.1109/tpds.2021.3138825","title":"Liquid: Intelligent Resource Estimation and Network-Efficient Scheduling for Deep Learning Jobs on Distributed GPU Clusters","display_name":"Liquid: Intelligent Resource Estimation and Network-Efficient Scheduling for Deep Learning Jobs on Distributed GPU Clusters","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W4206790491","doi":"https://doi.org/10.1109/tpds.2021.3138825"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2021.3138825","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2021.3138825","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052175650","display_name":"Rong Gu","orcid":"https://orcid.org/0000-0002-1565-9997"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rong Gu","raw_affiliation_strings":["Computer Science, Nanjing University, Nanjing, Jiangsu, China, (e-mail: gurong@nju.edu.cn)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science, Nanjing University, Nanjing, Jiangsu, China, (e-mail: gurong@nju.edu.cn)","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079049776","display_name":"Yuquan Chen","orcid":"https://orcid.org/0000-0001-5792-6566"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuquan Chen","raw_affiliation_strings":["Computer Science and Technology, Nanjing University, 12581 Nanjing, Jiangsu, China, (e-mail: yqchen@smail.nju.edu.cn)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science and Technology, Nanjing University, 12581 Nanjing, Jiangsu, China, (e-mail: yqchen@smail.nju.edu.cn)","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090788114","display_name":"Shuai Liu","orcid":"https://orcid.org/0000-0001-9909-0664"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuai Liu","raw_affiliation_strings":["Computer Science and Technology, Nanjing University, Nanjing, Jiangsu, China, (e-mail: newnius.cn@gmail.com)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science and Technology, Nanjing University, Nanjing, Jiangsu, China, (e-mail: newnius.cn@gmail.com)","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003181030","display_name":"Haipeng Dai","orcid":"https://orcid.org/0000-0003-0545-8187"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haipeng Dai","raw_affiliation_strings":["Computer Science and Technology, Nanjing University, 12581 Nanjing, Jiangsu, China, 210093 (e-mail: haipengdai@nju.edu.cn)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science and Technology, Nanjing University, 12581 Nanjing, Jiangsu, China, 210093 (e-mail: haipengdai@nju.edu.cn)","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100428808","display_name":"Guihai Chen","orcid":"https://orcid.org/0000-0002-6934-1685"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guihai Chen","raw_affiliation_strings":["Computer Science & Technology, Nanjing University, Nanjing, Jiangsu, China, 210093 (e-mail: gchen@nju.edu.cn)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science & Technology, Nanjing University, Nanjing, Jiangsu, China, 210093 (e-mail: gchen@nju.edu.cn)","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113211047","display_name":"Kai Zhang","orcid":"https://orcid.org/0009-0000-7342-1779"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kai Zhang","raw_affiliation_strings":["Alibaba Cloud, Alibaba Group, 518860 Hangzhou, Zhejiang, China, (e-mail: zk131586@alibaba-inc.com)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alibaba Cloud, Alibaba Group, 518860 Hangzhou, Zhejiang, China, (e-mail: zk131586@alibaba-inc.com)","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039682250","display_name":"Yang Che","orcid":"https://orcid.org/0009-0007-8220-1912"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yang Che","raw_affiliation_strings":["Alibaba Group, Alibaba Group, 518860 Hangzhou, Zhejiang, China, (e-mail: cheyang.cy@alibaba-inc.com)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alibaba Group, Alibaba Group, 518860 Hangzhou, Zhejiang, China, (e-mail: cheyang.cy@alibaba-inc.com)","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007538828","display_name":"Yihua Huang","orcid":"https://orcid.org/0000-0003-1806-0936"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yihua Huang","raw_affiliation_strings":["Computer Science & Technology, Nanjing University, Nanjing, Jiangsu, China, 210093 (e-mail: yhuang@nju.edu.cn)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science & Technology, Nanjing University, Nanjing, Jiangsu, China, 210093 (e-mail: yhuang@nju.edu.cn)","institution_ids":["https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":16.0859,"has_fulltext":false,"cited_by_count":75,"citation_normalized_percentile":{"value":0.99089161,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"1"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.877188503742218},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.6783702373504639},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.5556579828262329},{"id":"https://openalex.org/keywords/job-scheduler","display_name":"Job scheduler","score":0.5442916750907898},{"id":"https://openalex.org/keywords/shared-resource","display_name":"Shared resource","score":0.5379242897033691},{"id":"https://openalex.org/keywords/job-shop-scheduling","display_name":"Job shop scheduling","score":0.5359037518501282},{"id":"https://openalex.org/keywords/gpu-cluster","display_name":"GPU cluster","score":0.42652755975723267},{"id":"https://openalex.org/keywords/real-time-computing","display_name":"Real-time computing","score":0.36660104990005493},{"id":"https://openalex.org/keywords/schedule","display_name":"Schedule","score":0.20860528945922852},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.18184098601341248},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.16481122374534607},{"id":"https://openalex.org/keywords/cuda","display_name":"CUDA","score":0.1485423743724823},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.13958671689033508},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.11288416385650635},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.1087031364440918}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.877188503742218},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.6783702373504639},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5556579828262329},{"id":"https://openalex.org/C111873713","wikidata":"https://www.wikidata.org/wiki/Q1641413","display_name":"Job scheduler","level":3,"score":0.5442916750907898},{"id":"https://openalex.org/C51332947","wikidata":"https://www.wikidata.org/wiki/Q1172305","display_name":"Shared resource","level":2,"score":0.5379242897033691},{"id":"https://openalex.org/C55416958","wikidata":"https://www.wikidata.org/wiki/Q6206757","display_name":"Job shop scheduling","level":3,"score":0.5359037518501282},{"id":"https://openalex.org/C2781335571","wikidata":"https://www.wikidata.org/wiki/Q2633544","display_name":"GPU cluster","level":3,"score":0.42652755975723267},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.36660104990005493},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.20860528945922852},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.18184098601341248},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.16481122374534607},{"id":"https://openalex.org/C2778119891","wikidata":"https://www.wikidata.org/wiki/Q477690","display_name":"CUDA","level":2,"score":0.1485423743724823},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.13958671689033508},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.11288416385650635},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.1087031364440918},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2021.3138825","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2021.3138825","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5699999928474426,"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth"}],"awards":[{"id":"https://openalex.org/G1894088115","display_name":null,"funder_award_id":"61702254","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2475758149","display_name":"\u57fa\u4e8e\u8fb9\u7f18\u8ba1\u7b97\u7684\u4e91\u7aef\u878d\u5408\u7406\u8bba\u65b9\u6cd5\u4e0e\u5173\u952e\u6280\u672f\u7814\u7a76","funder_award_id":"61832005","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3624957052","display_name":"\u57fa\u4e8e\u667a\u80fd\u5316\u5206\u6790\u65b9\u6cd5\u7684\u5927\u6570\u636e\u7cfb\u7edf\u8f6f\u4ef6\u6027\u80fd\u4f18\u5316\u7814\u7a76","funder_award_id":"62072230","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5372072380","display_name":null,"funder_award_id":"U1811461","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":42,"referenced_works":["https://openalex.org/W1686810756","https://openalex.org/W2013067314","https://openalex.org/W2060393849","https://openalex.org/W2133693888","https://openalex.org/W2143408306","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2740920897","https://openalex.org/W2772774428","https://openalex.org/W2798515322","https://openalex.org/W2899071864","https://openalex.org/W2911964244","https://openalex.org/W2919594608","https://openalex.org/W2919897868","https://openalex.org/W2963403751","https://openalex.org/W2972392561","https://openalex.org/W3037681773","https://openalex.org/W3095922049","https://openalex.org/W3096403968","https://openalex.org/W3096484587","https://openalex.org/W3096583839","https://openalex.org/W3096956001","https://openalex.org/W3102873192","https://openalex.org/W3118608800","https://openalex.org/W3154142431","https://openalex.org/W3175707606","https://openalex.org/W4288630254","https://openalex.org/W6637373629","https://openalex.org/W6679901780","https://openalex.org/W6756009870","https://openalex.org/W6757922489","https://openalex.org/W6758283263","https://openalex.org/W6758474236","https://openalex.org/W6759814162","https://openalex.org/W6781728138","https://openalex.org/W6782839094","https://openalex.org/W6784425352","https://openalex.org/W6784871562","https://openalex.org/W6785115463","https://openalex.org/W6785197036","https://openalex.org/W6787972765","https://openalex.org/W6793903029"],"related_works":["https://openalex.org/W3204654320","https://openalex.org/W2139982839","https://openalex.org/W1984267569","https://openalex.org/W2092966558","https://openalex.org/W1956651153","https://openalex.org/W2347561926","https://openalex.org/W2327547880","https://openalex.org/W44553394","https://openalex.org/W3153517983","https://openalex.org/W4386430090"],"abstract_inverted_index":{"Deep":[0],"learning":[1],"(DL)":[2],"is":[3,25,48,77],"becoming":[4],"increasingly":[5],"popular":[6],"in":[7,38,102,158,226],"many":[8],"domains,":[9],"including":[10,179],"computer":[11],"vision,":[12],"speech":[13],"recognition,":[14],"self-driving":[15],"automobiles,":[16],"etc.":[17],"GPU":[18,32,53,117,184],"can":[19,195],"train":[20],"DL":[21,90,122],"models":[22],"efficiently":[23],"but":[24],"expensive,":[26],"which":[27,76],"motivates":[28],"users":[29,69,146],"to":[30,34,50,70,83,144],"share":[31],"resource":[33,54,74,85,118,126,141,168],"reduce":[35],"money":[36],"costs":[37],"practice.":[39],"To":[40,107],"ensure":[41],"efficient":[42,52,116],"sharing":[43],"among":[44],"multiple":[45],"users,":[46],"it":[47],"necessary":[49],"develop":[51],"management":[55,119],"and":[56,81,129,161,186,205],"scheduling":[57,89,156],"solutions.":[58,218],"However,":[59],"existing":[60],"ones":[61],"have":[62],"several":[63],"shortcomings.":[64],"First,":[65,131],"they":[66,92],"require":[67],"the":[68,72,95,109,166,197,207,220],"specify":[71],"job":[73,104,140,198,209],"requirement":[75,127,142,169],"usually":[78],"quite":[79],"inaccurate":[80],"leads":[82],"cluster":[84,96,154],"underutilization.":[86],"Second,":[87,150],"when":[88],"jobs,":[91],"rarely":[93],"take":[94],"network":[97],"characteristics":[98],"into":[99],"consideration,":[100],"resulting":[101],"low":[103],"execution":[105,199],"performance.":[106],"overcome":[108],"above":[110,167],"issues,":[111],"we":[112,132,151,173],"propose":[113,133,152,175],"Liquid,":[114],"an":[115],"platform":[120],"for":[121,139],"jobs":[123],"with":[124,216],"intelligent":[125,153],"estimation":[128,143,170],"scheduling.":[130],"a":[134],"regression":[135],"model":[136],"based":[137,164],"method":[138],"avoid":[145],"over-allocating":[147],"computing":[148],"resources.":[149],"network-efficient":[155],"methods":[157,223],"both":[159],"immediate":[160],"batch":[162],"modes":[163],"on":[165,203],"techniques.":[171],"Third,":[172],"further":[174],"three":[176],"system-level":[177],"optimizations,":[178],"pre-scheduling":[180],"data":[181],"transmission,":[182],"fine-grained":[183],"sharing,":[185],"event-driven":[187],"communication.":[188],"Experimental":[189],"results":[190],"show":[191],"that":[192],"our":[193],"Liquid":[194],"accelerate":[196],"speed":[200],"by":[201,213],"18%":[202],"average":[204,208],"shorten":[206],"completion":[210],"time":[211],"(JCT)":[212],"21%":[214],"compared":[215],"cutting-edge":[217],"Moreover,":[219],"proposed":[221],"optimization":[222],"are":[224],"effective":[225],"various":[227],"scenarios.":[228]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":17},{"year":2024,"cited_by_count":16},{"year":2023,"cited_by_count":23},{"year":2022,"cited_by_count":18}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}