{"id":"https://openalex.org/W4399666160","doi":"https://doi.org/10.1109/tcc.2024.3414440","title":"Non-Clairvoyant Scheduling of Distributed Machine Learning With Inter-Job and Intra-Job Parallelism on Heterogeneous GPUs","display_name":"Non-Clairvoyant Scheduling of Distributed Machine Learning With Inter-Job and Intra-Job Parallelism on Heterogeneous GPUs","publication_year":2024,"publication_date":"2024-06-14","ids":{"openalex":"https://openalex.org/W4399666160","doi":"https://doi.org/10.1109/tcc.2024.3414440"},"language":"en","primary_location":{"id":"doi:10.1109/tcc.2024.3414440","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcc.2024.3414440","pdf_url":null,"source":{"id":"https://openalex.org/S2492498579","display_name":"IEEE Transactions on Cloud Computing","issn_l":"2168-7161","issn":["2168-7161","2372-0018"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cloud Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088461411","display_name":"Fahao Chen","orcid":"https://orcid.org/0000-0002-4345-1296"},"institutions":[{"id":"https://openalex.org/I141591182","display_name":"University of Aizu","ror":"https://ror.org/02pg0e883","country_code":"JP","type":"education","lineage":["https://openalex.org/I141591182"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Fahao Chen","raw_affiliation_strings":["School of Computer Science and Engineering, University of Aizu, Aizuwakamatsu, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, University of Aizu, Aizuwakamatsu, Japan","institution_ids":["https://openalex.org/I141591182"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100432795","display_name":"Peng Li","orcid":"https://orcid.org/0000-0003-4981-0496"},"institutions":[{"id":"https://openalex.org/I141591182","display_name":"University of Aizu","ror":"https://ror.org/02pg0e883","country_code":"JP","type":"education","lineage":["https://openalex.org/I141591182"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Peng Li","raw_affiliation_strings":["School of Computer Science and Engineering, University of Aizu, Aizuwakamatsu, Japan"],"raw_orcid":"https://orcid.org/0000-0003-4981-0496","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, University of Aizu, Aizuwakamatsu, Japan","institution_ids":["https://openalex.org/I141591182"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046333931","display_name":"Celimuge Wu","orcid":"https://orcid.org/0000-0001-6853-5878"},"institutions":[{"id":"https://openalex.org/I20529979","display_name":"University of Electro-Communications","ror":"https://ror.org/02x73b849","country_code":"JP","type":"education","lineage":["https://openalex.org/I20529979"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Celimuge Wu","raw_affiliation_strings":["Meta-Networking Research Center, University of Electro-Communications, Chofu, Japan"],"raw_orcid":"https://orcid.org/0000-0001-6853-5878","affiliations":[{"raw_affiliation_string":"Meta-Networking Research Center, University of Electro-Communications, Chofu, Japan","institution_ids":["https://openalex.org/I20529979"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043464306","display_name":"Song Guo","orcid":"https://orcid.org/0000-0001-9831-2202"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Song Guo","raw_affiliation_strings":["Department of Computer Science and Engineering, Hong Kong University of Science and Technology, Hong Kong, SAR, China"],"raw_orcid":"https://orcid.org/0000-0001-9831-2202","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Hong Kong University of Science and Technology, Hong Kong, SAR, China","institution_ids":["https://openalex.org/I200769079"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2841,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.52536226,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"12","issue":"4","first_page":"1011","last_page":"1025"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7954314947128296},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.7062618732452393},{"id":"https://openalex.org/keywords/data-parallelism","display_name":"Data parallelism","score":0.6170297861099243},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.6089140772819519},{"id":"https://openalex.org/keywords/task-parallelism","display_name":"Task parallelism","score":0.5573997497558594},{"id":"https://openalex.org/keywords/symmetric-multiprocessor-system","display_name":"Symmetric multiprocessor system","score":0.5136048793792725},{"id":"https://openalex.org/keywords/job-scheduler","display_name":"Job scheduler","score":0.47743090987205505},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.4505314826965332},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.4297497570514679},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.15358152985572815}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7954314947128296},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.7062618732452393},{"id":"https://openalex.org/C61483411","wikidata":"https://www.wikidata.org/wiki/Q3124522","display_name":"Data parallelism","level":3,"score":0.6170297861099243},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.6089140772819519},{"id":"https://openalex.org/C42992933","wikidata":"https://www.wikidata.org/wiki/Q691169","display_name":"Task parallelism","level":3,"score":0.5573997497558594},{"id":"https://openalex.org/C172430144","wikidata":"https://www.wikidata.org/wiki/Q17111997","display_name":"Symmetric multiprocessor system","level":2,"score":0.5136048793792725},{"id":"https://openalex.org/C111873713","wikidata":"https://www.wikidata.org/wiki/Q1641413","display_name":"Job scheduler","level":3,"score":0.47743090987205505},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.4505314826965332},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.4297497570514679},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.15358152985572815},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tcc.2024.3414440","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcc.2024.3414440","pdf_url":null,"source":{"id":"https://openalex.org/S2492498579","display_name":"IEEE Transactions on Cloud Computing","issn_l":"2168-7161","issn":["2168-7161","2372-0018"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cloud Computing","raw_type":"journal-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-141393","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-141393","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6399999856948853,"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth"}],"awards":[{"id":"https://openalex.org/G1294907317","display_name":"\u504f\u5728\u8a08\u7b97\u8cc7\u6e90\u306e\u6d3b\u7528\u306b\u3088\u308b\u5de8\u5927\u30e2\u30c7\u30eb\u306b\u5bfe\u3059\u308b\u9023\u5408\u5b66\u7fd2\u30fb\u9023\u5408\u30a2\u30f3\u30e9\u30fc\u30cb\u30f3\u30b0\u6280\u8853\u306e\u78ba\u7acb","funder_award_id":"24K02932","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G3017900995","display_name":null,"funder_award_id":"JCYJ20200109142008673","funder_id":"https://openalex.org/F4320326705","funder_display_name":"Science, Technology and Innovation Commission of Shenzhen Municipality"},{"id":"https://openalex.org/G537960282","display_name":null,"funder_award_id":"23828673","funder_id":"https://openalex.org/F4320334789","funder_display_name":"Japan Science and Technology Agency"},{"id":"https://openalex.org/G7111805306","display_name":"\u9762\u5411\u8fb9\u7f18\u667a\u80fd\u7684\u8d44\u6e90\u914d\u7f6e\u4e0e\u90e8\u7f72\u4f18\u5316\u5173\u952e\u6280\u672f\u7814\u7a76","funder_award_id":"61872310","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320326705","display_name":"Science, Technology and Innovation Commission of Shenzhen Municipality","ror":null},{"id":"https://openalex.org/F4320334764","display_name":"Japan Society for the Promotion of Science","ror":"https://ror.org/00hhkn466"},{"id":"https://openalex.org/F4320334789","display_name":"Japan Science and Technology Agency","ror":"https://ror.org/00097mb19"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":40,"referenced_works":["https://openalex.org/W1543180643","https://openalex.org/W1922655562","https://openalex.org/W2073219414","https://openalex.org/W2083842231","https://openalex.org/W2121927366","https://openalex.org/W2149939304","https://openalex.org/W2150013062","https://openalex.org/W2183341477","https://openalex.org/W2189465200","https://openalex.org/W2194775991","https://openalex.org/W2579247884","https://openalex.org/W2798515322","https://openalex.org/W2896457183","https://openalex.org/W2962725887","https://openalex.org/W2963918968","https://openalex.org/W2969388332","https://openalex.org/W3006487403","https://openalex.org/W3022298203","https://openalex.org/W3022548332","https://openalex.org/W3047783725","https://openalex.org/W3091992486","https://openalex.org/W3092334294","https://openalex.org/W3209503812","https://openalex.org/W4283379731","https://openalex.org/W4290991121","https://openalex.org/W4301239768","https://openalex.org/W4385245566","https://openalex.org/W6637373629","https://openalex.org/W6640090968","https://openalex.org/W6687322159","https://openalex.org/W6738964360","https://openalex.org/W6748645090","https://openalex.org/W6748799445","https://openalex.org/W6756009870","https://openalex.org/W6759814162","https://openalex.org/W6769424276","https://openalex.org/W6781728138","https://openalex.org/W6782839094","https://openalex.org/W6784871562","https://openalex.org/W6785197036"],"related_works":["https://openalex.org/W2950520577","https://openalex.org/W1501159154","https://openalex.org/W2003935582","https://openalex.org/W74409296","https://openalex.org/W1554644772","https://openalex.org/W2494130044","https://openalex.org/W2468095077","https://openalex.org/W29548032","https://openalex.org/W2593878938","https://openalex.org/W305742777"],"abstract_inverted_index":{"Distributed":[0],"machine":[1],"learning":[2,26],"(DML)":[3],"has":[4,75],"shown":[5],"great":[6,87],"promise":[7],"in":[8,65,97,121],"accelerating":[9],"model":[10],"training":[11,145],"on":[12,45,70,187],"multiple":[13,25,71],"GPUs.":[14,46,60],"To":[15],"increase":[16],"GPU":[17,29,61,124],"utilization,":[18],"a":[19,110,122,130,144,158,200,204,217],"common":[20,64],"practice":[21],"is":[22,38,63,174],"to":[23,40,56,139,162],"let":[24],"jobs":[27,44,81],"share":[28],"clusters,":[30],"where":[31],"the":[32,98,164,188,214],"most":[33],"fundamental":[34],"and":[35,118,154,169,203],"critical":[36],"challenge":[37],"how":[39],"efficiently":[41],"schedule":[42],"these":[43],"However,":[47],"existing":[48],"works":[49],"about":[50,221],"DML":[51,72,80,111],"job":[52,73,112,151,166],"scheduling":[53,74],"are":[54],"constrained":[55],"settings":[57],"with":[58],"homogeneous":[59],"heterogeneity":[62],"practice,":[66],"but":[67],"its":[68,171],"influence":[69],"been":[76,94],"seldom":[77],"studied.":[78],"Moreover,":[79],"have":[82,91],"internal":[83],"structures":[84],"that":[85,114,135,210],"contain":[86],"parallelism":[88,120],"potentials,":[89],"which":[90],"not":[92],"yet":[93],"fully":[95],"exploited":[96],"heterogeneous":[99,123],"computing":[100],"environment.":[101],"In":[102],"this":[103],"paper,":[104],"we":[105,156,181],"propose":[106,157,182],"<italic":[107,126,196],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[108,127,197],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Hare</i>,":[109],"scheduler":[113],"exploits":[115],"both":[116],"inter-job":[117],"intra-job":[119],"cluster.":[125],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Hare</i>":[128,198],"adopts":[129],"relaxed":[131],"fixed-scale":[132],"synchronization":[133],"scheme":[134],"allows":[136],"independent":[137],"tasks":[138],"be":[140],"flexibly":[141],"scheduled":[142],"within":[143],"round.":[146],"Given":[147],"full":[148],"knowledge":[149,178],"of":[150,179,220],"arrival":[152],"time":[153,168],"sizes,":[155],"fast":[159],"heuristic":[160],"algorithm":[161,185],"minimize":[163],"average":[165],"completion":[167],"derive":[170],"theoretical":[172],"bound":[173],"derived.":[175],"Without":[176],"prior":[177],"jobs,":[180],"an":[183],"online":[184],"based":[186],"Heterogeneity-aware":[189],"Least-Attained":[190],"Service":[191],"(HLAS)":[192],"policy.":[193],"We":[194],"evaluate":[195],"using":[199],"small-scale":[201],"testbed":[202],"trace-driven":[205],"simulator.":[206],"The":[207],"results":[208],"show":[209],"it":[211],"can":[212],"outperform":[213],"state-of-the-art,":[215],"achieving":[216],"performance":[218],"improvement":[219],"2.94\u00d7.":[222]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2025-12-26T23:08:49.675405","created_date":"2025-10-10T00:00:00"}