{"id":"https://openalex.org/W2503654596","doi":"https://doi.org/10.1109/infocom.2016.7524415","title":"Symbiosis: Network-aware task scheduling in data-parallel frameworks","display_name":"Symbiosis: Network-aware task scheduling in data-parallel frameworks","publication_year":2016,"publication_date":"2016-04-01","ids":{"openalex":"https://openalex.org/W2503654596","doi":"https://doi.org/10.1109/infocom.2016.7524415","mag":"2503654596"},"language":"en","primary_location":{"id":"doi:10.1109/infocom.2016.7524415","is_oa":false,"landing_page_url":"https://doi.org/10.1109/infocom.2016.7524415","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE INFOCOM 2016 - The 35th Annual IEEE International Conference on Computer Communications","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033448563","display_name":"Jingjie Jiang","orcid":"https://orcid.org/0000-0002-2199-0301"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jingjie Jiang","raw_affiliation_strings":["Department of Computer Science and Engineering, Hong Kong University of Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Hong Kong University of Science and Technology","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028167318","display_name":"Shiyao Ma","orcid":"https://orcid.org/0000-0002-9013-789X"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shiyao Ma","raw_affiliation_strings":["Department of Computer Science and Engineering, Hong Kong University of Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Hong Kong University of Science and Technology","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100374506","display_name":"Bo Li","orcid":"https://orcid.org/0000-0003-2083-9105"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Bo Li","raw_affiliation_strings":["Department of Computer Science and Engineering, Hong Kong University of Science and Technology","University of Toronto, Toronto, ON, CA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Hong Kong University of Science and Technology","institution_ids":["https://openalex.org/I200769079"]},{"raw_affiliation_string":"University of Toronto, Toronto, ON, CA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5083596391","display_name":"Baochun Li","orcid":"https://orcid.org/0000-0003-2404-0974"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Baochun Li","raw_affiliation_strings":["Department of Computer Science and Engineering, Hong Kong University of Science and Technology","University of Toronto, Toronto, ON, CA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Hong Kong University of Science and Technology","institution_ids":["https://openalex.org/I200769079"]},{"raw_affiliation_string":"University of Toronto, Toronto, ON, CA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I200769079"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":27,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10829","display_name":"Interconnection Networks and Systems","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8208926916122437},{"id":"https://openalex.org/keywords/executor","display_name":"Executor","score":0.76435387134552},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.740247368812561},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.6890986561775208},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.6390464901924133},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.6084709167480469},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.47732239961624146},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.4561466872692108},{"id":"https://openalex.org/keywords/node","display_name":"Node (physics)","score":0.4502531588077545},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.4142891764640808},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.14297711849212646}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8208926916122437},{"id":"https://openalex.org/C180591056","wikidata":"https://www.wikidata.org/wiki/Q654437","display_name":"Executor","level":2,"score":0.76435387134552},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.740247368812561},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.6890986561775208},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.6390464901924133},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.6084709167480469},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.47732239961624146},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.4561466872692108},{"id":"https://openalex.org/C62611344","wikidata":"https://www.wikidata.org/wiki/Q1062658","display_name":"Node (physics)","level":2,"score":0.4502531588077545},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.4142891764640808},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.14297711849212646},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C66938386","wikidata":"https://www.wikidata.org/wiki/Q633538","display_name":"Structural engineering","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/infocom.2016.7524415","is_oa":false,"landing_page_url":"https://doi.org/10.1109/infocom.2016.7524415","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE INFOCOM 2016 - The 35th Annual IEEE International Conference on Computer Communications","raw_type":"proceedings-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-80583","is_oa":false,"landing_page_url":"http://www.scopus.com/record/display.url?eid=2-s2.0-84983314445&origin=inward","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference paper"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6100000143051147,"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W1698388015","https://openalex.org/W1746258828","https://openalex.org/W1854214752","https://openalex.org/W1876812886","https://openalex.org/W1992710140","https://openalex.org/W1994035478","https://openalex.org/W2000444555","https://openalex.org/W2043566022","https://openalex.org/W2091520147","https://openalex.org/W2096125134","https://openalex.org/W2102849319","https://openalex.org/W2110514750","https://openalex.org/W2120422789","https://openalex.org/W2131975293","https://openalex.org/W2149939304","https://openalex.org/W2163961697","https://openalex.org/W2168427242","https://openalex.org/W2169246522","https://openalex.org/W2170616854","https://openalex.org/W2173213060","https://openalex.org/W2324866405","https://openalex.org/W3160520312","https://openalex.org/W4251459357","https://openalex.org/W4255119735","https://openalex.org/W6637806892","https://openalex.org/W6639324668","https://openalex.org/W6679815717","https://openalex.org/W6684084819"],"related_works":["https://openalex.org/W3132876088","https://openalex.org/W4237320244","https://openalex.org/W3107299409","https://openalex.org/W3044912482","https://openalex.org/W3015007115","https://openalex.org/W3016598040","https://openalex.org/W2230606172","https://openalex.org/W2603352950","https://openalex.org/W4241425178","https://openalex.org/W4246956338"],"abstract_inverted_index":{"Even":[0],"with":[1,187],"the":[2,16,56,132],"recent":[3],"proliferation":[4],"of":[5,58,71],"in-memory":[6],"computation":[7,48,59],"in":[8,95,104,131,181],"data-parallel":[9,152],"frameworks":[10],"(such":[11],"as":[12,27,30,42],"Spark),":[13],"transfers":[14,25],"over":[15],"network":[17,24,50,61],"are":[18,102],"still":[19],"time-consuming.":[20],"Similar":[21],"to":[22,33,67,89,143,149,183],"computation,":[23],"serve":[26],"main":[28],"roadblocks":[29],"we":[31,78],"try":[32],"minimize":[34],"job":[35,176],"completion":[36,177],"times.":[37],"Existing":[38],"schedulers":[39],"were":[40],"designed":[41,88],"isolated":[43],"solutions":[44],"that":[45,100,114,173],"focused":[46],"on":[47,165],"or":[49],"performance":[51],"only.":[52],"Without":[53],"any":[54],"coordination,":[55],"utilization":[57],"and":[60,81,92,121,128,148,160],"resources":[62,101],"may":[63],"become":[64],"unbalanced,":[65],"leading":[66],"a":[68,84,96,105,137,166],"reduced":[69],"level":[70],"overall":[72],"resource":[73,116],"utilization.":[74],"In":[75],"this":[76],"paper,":[77],"design,":[79],"implement,":[80],"evaluate":[82],"Symbiosis,":[83],"network-aware":[85],"task":[86],"scheduler":[87,113,186],"coordinate":[90],"computation-bound":[91,127],"network-bound":[93,129],"tasks":[94,130],"large":[97],"cluster,":[98],"so":[99],"utilized":[103],"more":[106],"balanced":[107],"fashion.":[108],"Symbiosis":[109,157,174],"is":[110,141],"an":[111],"online":[112],"predicts":[115],"imbalance":[117,124],"before":[118],"launching":[119],"tasks,":[120],"correct":[122],"such":[123],"by":[125,179],"co-locating":[126],"same":[133],"executor":[134],"process.":[135],"As":[136],"guiding":[138],"principle,":[139],"it":[140],"engineered":[142],"be":[144],"practically":[145],"implemented":[146,156],"within":[147,158],"complement":[150],"existing":[151],"frameworks.":[153],"We":[154,169],"have":[155],"Spark,":[159],"carried":[161],"out":[162],"our":[163],"experiments":[164],"100-node":[167],"cluster.":[168],"show":[170],"convincing":[171],"evidence":[172],"reduces":[175],"times":[178],"11.9%":[180],"comparison":[182],"Spark's":[184],"current":[185],"little":[188],"overhead.":[189]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":4}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}