{"id":"https://openalex.org/W2889455461","doi":"https://doi.org/10.1109/tpds.2018.2866993","title":"Dependency-Aware Network Adaptive Scheduling of Data-Intensive Parallel Jobs","display_name":"Dependency-Aware Network Adaptive Scheduling of Data-Intensive Parallel Jobs","publication_year":2018,"publication_date":"2018-08-27","ids":{"openalex":"https://openalex.org/W2889455461","doi":"https://doi.org/10.1109/tpds.2018.2866993","mag":"2889455461"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2018.2866993","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2018.2866993","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070913208","display_name":"Shaoqi Wang","orcid":"https://orcid.org/0000-0002-8187-779X"},"institutions":[{"id":"https://openalex.org/I888729015","display_name":"University of Colorado Colorado Springs","ror":"https://ror.org/054spjc55","country_code":"US","type":"education","lineage":["https://openalex.org/I888729015"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Shaoqi Wang","raw_affiliation_strings":["Department of Computer Science, University of Colorado, Colorado Springs, CO, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Colorado, Colorado Springs, CO, USA","institution_ids":["https://openalex.org/I888729015"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100344511","display_name":"Wei Chen","orcid":"https://orcid.org/0000-0002-9402-6249"},"institutions":[{"id":"https://openalex.org/I888729015","display_name":"University of Colorado Colorado Springs","ror":"https://ror.org/054spjc55","country_code":"US","type":"education","lineage":["https://openalex.org/I888729015"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wei Chen","raw_affiliation_strings":["Department of Computer Science, University of Colorado, Colorado Springs, CO, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Colorado, Colorado Springs, CO, USA","institution_ids":["https://openalex.org/I888729015"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078528016","display_name":"Xiaobo Zhou","orcid":"https://orcid.org/0000-0002-0466-7609"},"institutions":[{"id":"https://openalex.org/I888729015","display_name":"University of Colorado Colorado Springs","ror":"https://ror.org/054spjc55","country_code":"US","type":"education","lineage":["https://openalex.org/I888729015"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaobo Zhou","raw_affiliation_strings":["Department of Computer Science, University of Colorado, Colorado Springs, CO, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Colorado, Colorado Springs, CO, USA","institution_ids":["https://openalex.org/I888729015"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000194231","display_name":"Liqiang Zhang","orcid":"https://orcid.org/0000-0001-8412-8391"},"institutions":[{"id":"https://openalex.org/I4210134870","display_name":"Indiana University South Bend","ror":"https://ror.org/02s1hvj37","country_code":"US","type":"education","lineage":["https://openalex.org/I4210134870","https://openalex.org/I592451"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liqiang Zhang","raw_affiliation_strings":["Department of Computer & Information Sciences, Indiana University South Bend, South Bend, IN, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer & Information Sciences, Indiana University South Bend, South Bend, IN, USA","institution_ids":["https://openalex.org/I4210134870"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5011242335","display_name":"Yin Wang","orcid":"https://orcid.org/0000-0003-3274-0452"},"institutions":[{"id":"https://openalex.org/I116953780","display_name":"Tongji University","ror":"https://ror.org/03rc6as71","country_code":"CN","type":"education","lineage":["https://openalex.org/I116953780"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yin Wang","raw_affiliation_strings":["Department of Compute Science, Tongji University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Department of Compute Science, Tongji University, Shanghai, China","institution_ids":["https://openalex.org/I116953780"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5070913208"],"corresponding_institution_ids":["https://openalex.org/I888729015"],"apc_list":null,"apc_paid":null,"fwci":6.1623,"has_fulltext":false,"cited_by_count":21,"citation_normalized_percentile":{"value":0.96666137,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"30","issue":"3","first_page":"515","last_page":"529"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9934999942779541,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8972682952880859},{"id":"https://openalex.org/keywords/locality","display_name":"Locality","score":0.6868690848350525},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.5784335136413574},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.5689865350723267},{"id":"https://openalex.org/keywords/job-scheduler","display_name":"Job scheduler","score":0.5611568093299866},{"id":"https://openalex.org/keywords/schedule","display_name":"Schedule","score":0.5472620725631714},{"id":"https://openalex.org/keywords/dependency","display_name":"Dependency (UML)","score":0.5071420669555664},{"id":"https://openalex.org/keywords/yarn","display_name":"Yarn","score":0.43774792551994324},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.28738313913345337},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.16982445120811462},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.12274444103240967}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8972682952880859},{"id":"https://openalex.org/C2779808786","wikidata":"https://www.wikidata.org/wiki/Q6664603","display_name":"Locality","level":2,"score":0.6868690848350525},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5784335136413574},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.5689865350723267},{"id":"https://openalex.org/C111873713","wikidata":"https://www.wikidata.org/wiki/Q1641413","display_name":"Job scheduler","level":3,"score":0.5611568093299866},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.5472620725631714},{"id":"https://openalex.org/C19768560","wikidata":"https://www.wikidata.org/wiki/Q320727","display_name":"Dependency (UML)","level":2,"score":0.5071420669555664},{"id":"https://openalex.org/C2778787235","wikidata":"https://www.wikidata.org/wiki/Q49007","display_name":"Yarn","level":2,"score":0.43774792551994324},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.28738313913345337},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.16982445120811462},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.12274444103240967},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C160403385","wikidata":"https://www.wikidata.org/wiki/Q220543","display_name":"Queue","level":2,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tpds.2018.2866993","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2018.2866993","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6299999952316284,"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":55,"referenced_works":["https://openalex.org/W1420267637","https://openalex.org/W1471136644","https://openalex.org/W1487866729","https://openalex.org/W1550206324","https://openalex.org/W1854214752","https://openalex.org/W1880262756","https://openalex.org/W1971596129","https://openalex.org/W1998471240","https://openalex.org/W2007601752","https://openalex.org/W2040728701","https://openalex.org/W2048110401","https://openalex.org/W2054141820","https://openalex.org/W2074988668","https://openalex.org/W2075620950","https://openalex.org/W2076406848","https://openalex.org/W2094469165","https://openalex.org/W2096125134","https://openalex.org/W2097484305","https://openalex.org/W2104993419","https://openalex.org/W2105947650","https://openalex.org/W2110086534","https://openalex.org/W2119565742","https://openalex.org/W2129542763","https://openalex.org/W2131975293","https://openalex.org/W2154138844","https://openalex.org/W2161160262","https://openalex.org/W2173213060","https://openalex.org/W2299310877","https://openalex.org/W2400388873","https://openalex.org/W2418529204","https://openalex.org/W2465698270","https://openalex.org/W2465838301","https://openalex.org/W2521597241","https://openalex.org/W2528158958","https://openalex.org/W2552554409","https://openalex.org/W2576355797","https://openalex.org/W2626230533","https://openalex.org/W2724721453","https://openalex.org/W2740985301","https://openalex.org/W2742956305","https://openalex.org/W2744319535","https://openalex.org/W2886715217","https://openalex.org/W2925286065","https://openalex.org/W2999590865","https://openalex.org/W4239670432","https://openalex.org/W6628579693","https://openalex.org/W6632865047","https://openalex.org/W6639055396","https://openalex.org/W6639619044","https://openalex.org/W6679815717","https://openalex.org/W6697320668","https://openalex.org/W6697525719","https://openalex.org/W6727321242","https://openalex.org/W6732337852","https://openalex.org/W6741589387"],"related_works":["https://openalex.org/W2160425906","https://openalex.org/W1882733036","https://openalex.org/W3004264150","https://openalex.org/W2546696010","https://openalex.org/W1992741870","https://openalex.org/W2025394122","https://openalex.org/W2109998134","https://openalex.org/W2343243188","https://openalex.org/W2014963843","https://openalex.org/W2085797995"],"abstract_inverted_index":{"Datacenter":[0],"clusters":[1,176],"often":[2,21],"run":[3],"data-intensive":[4],"jobs":[5,19,51,71,214,222],"in":[6,125,155],"parallel":[7,18],"for":[8,76,221],"improving":[9],"resource":[10],"utilization":[11],"and":[12,49,63,67,92,104,142,169,174,181,196,203],"cost":[13],"efficiency.":[14],"The":[15,109,132],"performance":[16,87,212],"of":[17,41,69,88,139,160,213],"is":[20],"constrained":[22],"by":[23,192],"the":[24,37,65,86,137,152,211],"cluster's":[25],"hard-to-scale":[26],"network":[27,144],"bisection":[28],"bandwidth.":[29],"Various":[30],"solutions":[31],"have":[32],"been":[33],"proposed":[34],"to":[35,83,122,127,149,157,194,200],"address":[36],"issue,":[38],"however,":[39],"most":[40],"them":[42],"do":[43],"not":[44,207],"consider":[45],"inter-job":[46],"data":[47,66,77,154,161],"dependencies":[48,114],"schedule":[50],"independently":[52],"from":[53],"one":[54],"another.":[55],"In":[56],"this":[57],"work,":[58],"we":[59],"find":[60],"that":[61,80,99,186],"aggregating":[62],"co-locating":[64],"tasks":[68,124,148],"dependent":[70,130,153],"offer":[72],"an":[73,101,105],"extra":[74],"opportunity":[75],"locality":[78],"improvement":[79],"can":[81],"help":[82],"greatly":[84],"enhance":[85],"jobs.":[89],"We":[90,163],"propose":[91],"design":[93],"Dawn,":[94],"a":[95],"dependency-aware":[96],"network-adaptive":[97],"scheduler":[98],"includes":[100],"online":[102,110,140],"plan":[103,141],"adaptive":[106],"task":[107,133],"scheduler.":[108],"plan,":[111],"taking":[112],"job":[113],"into":[115],"consideration,":[116],"determines":[117],"where":[118],"(i.e.,":[119],"preferred":[120],"racks)":[121],"place":[123],"order":[126,156],"proactively":[128],"aggregate":[129],"data.":[131],"scheduler,":[134],"based":[135],"on":[136,166,172],"output":[138],"dynamic":[143],"status,":[145],"adaptively":[146],"schedules":[147],"co-locate":[150],"with":[151,215],"take":[158],"advantage":[159],"locality.":[162],"implement":[164],"Dawn":[165,187,206],"Apache":[167],"Yarn":[168],"evaluate":[170],"it":[171],"physical":[173],"virtual":[175],"using":[177],"various":[178],"machine":[179],"learning":[180],"query":[182],"workloads.":[183],"Results":[184],"show":[185],"effectively":[188],"improves":[189],"cluster":[190],"throughput":[191],"up":[193],"73":[195],"38":[197],"percent":[198],"compared":[199],"Fair":[201],"Scheduler":[202],"ShuffleWatcher,":[204],"respectively.":[205],"only":[208],"significantly":[209],"enhances":[210],"dependency,":[216],"but":[217],"also":[218],"works":[219],"well":[220],"without":[223],"dependency.":[224]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":6},{"year":2018,"cited_by_count":1}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}