{"id":"https://openalex.org/W2074988668","doi":"https://doi.org/10.1145/2785956.2787488","title":"Network-Aware Scheduling for Data-Parallel Jobs","display_name":"Network-Aware Scheduling for Data-Parallel Jobs","publication_year":2015,"publication_date":"2015-08-17","ids":{"openalex":"https://openalex.org/W2074988668","doi":"https://doi.org/10.1145/2785956.2787488","mag":"2074988668"},"language":"en","primary_location":{"id":"doi:10.1145/2785956.2787488","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2785956.2787488","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2015 ACM Conference on Special Interest Group on Data Communication","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064636812","display_name":"Virajith Jalaparti","orcid":null},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Virajith Jalaparti","raw_affiliation_strings":["University of Illinois, Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois, Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044047156","display_name":"Peter Bod\u00edk","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peter Bodik","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032872922","display_name":"Ishai Menache","orcid":"https://orcid.org/0000-0002-2540-236X"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ishai Menache","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070183492","display_name":"Sriram Rao","orcid":"https://orcid.org/0000-0002-3265-6165"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sriram Rao","raw_affiliation_strings":["Microsoft, Redmond, WA, USA","Microsoft Redmond, WA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft Redmond, WA, USA#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033681881","display_name":"Konstantin Makarychev","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Konstantin Makarychev","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5038897777","display_name":"Matthew Caesar","orcid":"https://orcid.org/0000-0001-5955-9229"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew Caesar","raw_affiliation_strings":["University of Illinois, Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois, Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5064636812"],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":null,"apc_paid":null,"fwci":73.9826,"has_fulltext":false,"cited_by_count":156,"citation_normalized_percentile":{"value":0.99917274,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"407","last_page":"420"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8416023254394531},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.7275024652481079},{"id":"https://openalex.org/keywords/locality","display_name":"Locality","score":0.7006886601448059},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.6591135859489441},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.6145948171615601},{"id":"https://openalex.org/keywords/yarn","display_name":"Yarn","score":0.5409567952156067},{"id":"https://openalex.org/keywords/schedule","display_name":"Schedule","score":0.4825010299682617},{"id":"https://openalex.org/keywords/job-shop-scheduling","display_name":"Job shop scheduling","score":0.47544312477111816},{"id":"https://openalex.org/keywords/job-scheduler","display_name":"Job scheduler","score":0.43827542662620544},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.3563758134841919},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.34388357400894165},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.2252812683582306},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.12563470005989075}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8416023254394531},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.7275024652481079},{"id":"https://openalex.org/C2779808786","wikidata":"https://www.wikidata.org/wiki/Q6664603","display_name":"Locality","level":2,"score":0.7006886601448059},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.6591135859489441},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.6145948171615601},{"id":"https://openalex.org/C2778787235","wikidata":"https://www.wikidata.org/wiki/Q49007","display_name":"Yarn","level":2,"score":0.5409567952156067},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.4825010299682617},{"id":"https://openalex.org/C55416958","wikidata":"https://www.wikidata.org/wiki/Q6206757","display_name":"Job shop scheduling","level":3,"score":0.47544312477111816},{"id":"https://openalex.org/C111873713","wikidata":"https://www.wikidata.org/wiki/Q1641413","display_name":"Job scheduler","level":3,"score":0.43827542662620544},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.3563758134841919},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.34388357400894165},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.2252812683582306},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.12563470005989075},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C160403385","wikidata":"https://www.wikidata.org/wiki/Q220543","display_name":"Queue","level":2,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/2785956.2787488","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2785956.2787488","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2015 ACM Conference on Special Interest Group on Data Communication","raw_type":"proceedings-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.708.8745","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.708.8745","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://web.engr.illinois.edu/%7Ejalapar1/papers/corral_sigcomm15_final.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.41999998688697815}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":38,"referenced_works":["https://openalex.org/W175711003","https://openalex.org/W1471136644","https://openalex.org/W1535431348","https://openalex.org/W1579147384","https://openalex.org/W1834532152","https://openalex.org/W1975231449","https://openalex.org/W1984116073","https://openalex.org/W2004958263","https://openalex.org/W2022678927","https://openalex.org/W2024077218","https://openalex.org/W2025549137","https://openalex.org/W2040466547","https://openalex.org/W2041512590","https://openalex.org/W2048554864","https://openalex.org/W2057913648","https://openalex.org/W2086190489","https://openalex.org/W2092086632","https://openalex.org/W2094469165","https://openalex.org/W2096125134","https://openalex.org/W2099602864","https://openalex.org/W2101871381","https://openalex.org/W2102294813","https://openalex.org/W2102549685","https://openalex.org/W2104680817","https://openalex.org/W2110086534","https://openalex.org/W2110514750","https://openalex.org/W2119400430","https://openalex.org/W2122465391","https://openalex.org/W2133569115","https://openalex.org/W2140509629","https://openalex.org/W2149939304","https://openalex.org/W2151251992","https://openalex.org/W2154042331","https://openalex.org/W2163291889","https://openalex.org/W2168427242","https://openalex.org/W2189465200","https://openalex.org/W6632229964","https://openalex.org/W6687322159"],"related_works":["https://openalex.org/W2078851640","https://openalex.org/W2381070915","https://openalex.org/W2139982839","https://openalex.org/W1984267569","https://openalex.org/W2092966558","https://openalex.org/W1956651153","https://openalex.org/W2347561926","https://openalex.org/W2327547880","https://openalex.org/W2014963843","https://openalex.org/W3153517983"],"abstract_inverted_index":{"To":[0],"reduce":[1],"the":[2,29,37,67,96,143,174,183],"impact":[3],"of":[4,25,50,69,73,110,142,176],"network":[5,22,81],"congestion":[6],"on":[7,95,153,159],"big":[8],"data":[9,32,70,122,128,194],"jobs,":[10],"cluster":[11,163],"management":[12],"frameworks":[13],"use":[14],"various":[15],"heuristics":[16],"to":[17,44,61,113,125,168,180,188],"schedule":[18,36,117],"compute":[19,124],"tasks":[20,38,72],"and/or":[21],"flows.":[23],"Most":[24],"these":[26,74,177],"schedulers":[27],"consider":[28],"job":[30],"input":[31],"fixed":[33],"and":[34,39,71,83,123,130,145,156,182],"greedily":[35],"flows":[40],"that":[41,107],"are":[42,53],"ready":[43],"run.":[45],"However,":[46],"a":[47,104,160],"large":[48],"fraction":[49],"production":[51,165],"jobs":[52,75,93,133],"recurring":[54],"with":[55,190],"predictable":[56],"characteristics,":[57],"which":[58,87,118],"allows":[59,76],"us":[60],"plan":[62],"ahead":[63],"for":[64,77],"them.":[65],"Coordinating":[66],"placement":[68],"significantly":[78],"improving":[79,147],"their":[80,148],"locality":[82],"freeing":[84],"up":[85,179,187],"bandwidth,":[86],"can":[88],"be":[89],"used":[90],"by":[91],"other":[92],"running":[94],"cluster.":[97],"With":[98],"this":[99],"intuition,":[100],"we":[101],"develop":[102],"Corral,":[103],"scheduling":[105,137],"framework":[106],"uses":[108],"characteristics":[109],"future":[111],"workloads":[112,178],"determine":[114],"an":[115],"offline":[116],"(i)":[119],"jointly":[120],"places":[121],"achieve":[126],"better":[127],"locality,":[129],"(ii)":[131],"isolates":[132],"both":[134],"spatially":[135],"(by":[136],"them":[138],"in":[139,193],"different":[140],"parts":[141],"cluster)":[144],"temporally,":[146],"performance.":[149],"We":[150],"implement":[151],"Corral":[152,172],"Apache":[154],"Yarn,":[155],"evaluate":[157],"it":[158],"210":[161],"machine":[162],"using":[164],"workloads.":[166],"Compared":[167],"Yarn's":[169],"capacity":[170],"scheduler,":[171],"reduces":[173],"makespan":[175],"33%":[181],"median":[184],"completion":[185],"time":[186],"56%,":[189],"20-90%":[191],"reduction":[192],"transferred":[195],"across":[196],"racks.":[197]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":15},{"year":2020,"cited_by_count":13},{"year":2019,"cited_by_count":22},{"year":2018,"cited_by_count":35},{"year":2017,"cited_by_count":26},{"year":2016,"cited_by_count":27},{"year":2015,"cited_by_count":2}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}