{"id":"https://openalex.org/W2911441061","doi":"https://doi.org/10.1109/tcc.2019.2894779","title":"New Scheduling Algorithms for Improving Performance and Resource Utilization in Hadoop YARN Clusters","display_name":"New Scheduling Algorithms for Improving Performance and Resource Utilization in Hadoop YARN Clusters","publication_year":2019,"publication_date":"2019-01-24","ids":{"openalex":"https://openalex.org/W2911441061","doi":"https://doi.org/10.1109/tcc.2019.2894779","mag":"2911441061"},"language":"en","primary_location":{"id":"doi:10.1109/tcc.2019.2894779","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcc.2019.2894779","pdf_url":null,"source":{"id":"https://openalex.org/S2492498579","display_name":"IEEE Transactions on Cloud Computing","issn_l":"2168-7161","issn":["2168-7161","2372-0018"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cloud Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052203401","display_name":"Yi Yao","orcid":"https://orcid.org/0000-0003-3582-5726"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Yi Yao","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Northeastern University, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Northeastern University, Boston, MA, USA","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100644401","display_name":"Han Gao","orcid":"https://orcid.org/0000-0003-2399-5007"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Han Gao","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Northeastern University, Boston, MA, USA"],"raw_orcid":"https://orcid.org/0000-0003-2399-5007","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Northeastern University, Boston, MA, USA","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101584767","display_name":"Jiayin Wang","orcid":"https://orcid.org/0000-0001-5103-6132"},"institutions":[{"id":"https://openalex.org/I166088655","display_name":"Montclair State University","ror":"https://ror.org/01nxc2t48","country_code":"US","type":"education","lineage":["https://openalex.org/I166088655"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiayin Wang","raw_affiliation_strings":["Department of Computer Science, Montclair State University, Montclair, NJ, USA"],"raw_orcid":"https://orcid.org/0000-0001-5103-6132","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Montclair State University, Montclair, NJ, USA","institution_ids":["https://openalex.org/I166088655"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085815921","display_name":"Bo Sheng","orcid":"https://orcid.org/0000-0001-5958-8935"},"institutions":[{"id":"https://openalex.org/I33434090","display_name":"University of Massachusetts Boston","ror":"https://ror.org/04ydmy275","country_code":"US","type":"education","lineage":["https://openalex.org/I33434090"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bo Sheng","raw_affiliation_strings":["Department of Computer Science, University of Massachusetts Boston, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Massachusetts Boston, Boston, MA, USA","institution_ids":["https://openalex.org/I33434090"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5091825452","display_name":"Ningfang Mi","orcid":"https://orcid.org/0009-0005-0934-1287"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ningfang Mi","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Northeastern University, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Northeastern University, Boston, MA, USA","institution_ids":["https://openalex.org/I12912129"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5052203401"],"corresponding_institution_ids":["https://openalex.org/I12912129"],"apc_list":null,"apc_paid":null,"fwci":6.6947,"has_fulltext":false,"cited_by_count":45,"citation_normalized_percentile":{"value":0.96766898,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"9","issue":"3","first_page":"1158","last_page":"1171"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.9846000075340271,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/yarn","display_name":"Yarn","score":0.9156451225280762},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8220223188400269},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7003895044326782},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.6971749663352966},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.6562650799751282},{"id":"https://openalex.org/keywords/job-scheduler","display_name":"Job scheduler","score":0.5529918670654297},{"id":"https://openalex.org/keywords/job-shop-scheduling","display_name":"Job shop scheduling","score":0.5517942905426025},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.4266805946826935},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.30121272802352905},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.20202624797821045},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.10566306114196777},{"id":"https://openalex.org/keywords/schedule","display_name":"Schedule","score":0.08936318755149841},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07644140720367432}],"concepts":[{"id":"https://openalex.org/C2778787235","wikidata":"https://www.wikidata.org/wiki/Q49007","display_name":"Yarn","level":2,"score":0.9156451225280762},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8220223188400269},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7003895044326782},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.6971749663352966},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.6562650799751282},{"id":"https://openalex.org/C111873713","wikidata":"https://www.wikidata.org/wiki/Q1641413","display_name":"Job scheduler","level":3,"score":0.5529918670654297},{"id":"https://openalex.org/C55416958","wikidata":"https://www.wikidata.org/wiki/Q6206757","display_name":"Job shop scheduling","level":3,"score":0.5517942905426025},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.4266805946826935},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.30121272802352905},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.20202624797821045},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.10566306114196777},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.08936318755149841},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07644140720367432},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tcc.2019.2894779","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcc.2019.2894779","pdf_url":null,"source":{"id":"https://openalex.org/S2492498579","display_name":"IEEE Transactions on Cloud Computing","issn_l":"2168-7161","issn":["2168-7161","2372-0018"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cloud Computing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5600000023841858,"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth"}],"awards":[{"id":"https://openalex.org/G2692146287","display_name":null,"funder_award_id":"FA9550-14-1-0160","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G4543026422","display_name":null,"funder_award_id":"CNS 109253","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8883205329","display_name":null,"funder_award_id":"CNS-1452751","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320338279","display_name":"Air Force Office of Scientific Research","ror":"https://ror.org/011e9bt93"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":38,"referenced_works":["https://openalex.org/W122824882","https://openalex.org/W185167974","https://openalex.org/W872672612","https://openalex.org/W1890643295","https://openalex.org/W1902364784","https://openalex.org/W1968160647","https://openalex.org/W1969109994","https://openalex.org/W1970176495","https://openalex.org/W1976860187","https://openalex.org/W1992928432","https://openalex.org/W2002472616","https://openalex.org/W2013344760","https://openalex.org/W2022167206","https://openalex.org/W2022678927","https://openalex.org/W2024742034","https://openalex.org/W2038345112","https://openalex.org/W2039806905","https://openalex.org/W2057253279","https://openalex.org/W2058047606","https://openalex.org/W2083983420","https://openalex.org/W2096125134","https://openalex.org/W2122465391","https://openalex.org/W2133569115","https://openalex.org/W2137560895","https://openalex.org/W2137726616","https://openalex.org/W2138543065","https://openalex.org/W2230666400","https://openalex.org/W2470835488","https://openalex.org/W2496453903","https://openalex.org/W2497069692","https://openalex.org/W2549755022","https://openalex.org/W3162118826","https://openalex.org/W4247267780","https://openalex.org/W4250886557","https://openalex.org/W6639249596","https://openalex.org/W6689360179","https://openalex.org/W6720433847","https://openalex.org/W6729142878"],"related_works":["https://openalex.org/W3204654320","https://openalex.org/W2092966558","https://openalex.org/W1956651153","https://openalex.org/W1984267569","https://openalex.org/W2347561926","https://openalex.org/W2315766899","https://openalex.org/W2327547880","https://openalex.org/W2002944882","https://openalex.org/W44553394","https://openalex.org/W3153517983"],"abstract_inverted_index":{"The":[0,18,192],"MapReduce":[1,129,152,190],"framework":[2],"has":[3,21],"become":[4],"the":[5,62,70,83,92,118,121,137,162,167,172,202],"defacto":[6],"scheme":[7],"for":[8,35,91],"scalable":[9],"semi-structured":[10],"and":[11,40,56,78,144,182,204],"un-structured":[12],"data":[13],"processing":[14],"in":[15,46,52,65,103,131,151,166,180],"recent":[16],"years.":[17],"Hadoop":[19,27,132],"ecosystem":[20],"evolved":[22],"into":[23],"its":[24],"second":[25],"generation,":[26],"YARN,":[28],"which":[29,87,114],"adopts":[30],"fine-grained":[31],"resource":[32,48,72,95,142,206],"management":[33,49],"schemes":[34],"job":[36,101,163],"scheduling.":[37,80],"Nowadays,":[38],"fairness":[39],"efficiency":[41,93],"are":[42,54],"two":[43],"main":[44],"concerns":[45],"YARN":[47,53,66,112,133,181,198],"because":[50],"resources":[51,77],"shared":[55],"contended":[57],"by":[58,135,159],"multiple":[59],"applications.":[60],"However,":[61],"current":[63],"scheduling":[64,168,174],"does":[67],"not":[68],"yield":[69],"optimal":[71],"arrangement,":[73],"unnecessarily":[74],"causing":[75],"idle":[76],"inefficient":[79],"It":[81],"omits":[82],"dependency":[84,145],"between":[85,146],"tasks":[86],"is":[88],"extremely":[89],"crucial":[90],"of":[94,125,128,139,188],"utilization":[96],"as":[97,99,176],"well":[98],"heterogeneous":[100],"features":[102],"real":[104],"application":[105],"environments.":[106],"We":[107,170],"thus":[108],"propose":[109],"a":[110,126,177,186],"new":[111,173],"scheduler":[113,158,179,199],"can":[115],"effectively":[116,200],"reduce":[117],"makespan":[119],"(i.e.,":[120],"total":[122],"execution":[123],"time)":[124],"batch":[127],"jobs":[130],"clusters":[134],"leveraging":[136],"information":[138,165],"requested":[140],"resources,":[141],"capacities":[143],"tasks.":[147],"For":[148],"accommodating":[149],"heterogeneity":[150],"jobs,":[153],"we":[154],"also":[155],"extend":[156],"our":[157,197],"further":[160],"considering":[161],"iteration":[164],"decisions.":[169],"implemented":[171],"algorithm":[175],"pluggable":[178],"evaluated":[183],"it":[184],"with":[185],"set":[187],"classic":[189],"benchmarks.":[191],"experimental":[193],"results":[194],"demonstrate":[195],"that":[196],"reduces":[201],"makespans":[203],"improves":[205],"utilizations.":[207]},"counts_by_year":[{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":10},{"year":2022,"cited_by_count":10},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":6},{"year":2019,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}