{"id":"https://openalex.org/W2915653915","doi":"https://doi.org/10.1109/padsw.2018.8644650","title":"H-Scheduler: Storage-Aware Task Scheduling for Heterogeneous-Storage Spark Clusters","display_name":"H-Scheduler: Storage-Aware Task Scheduling for Heterogeneous-Storage Spark Clusters","publication_year":2018,"publication_date":"2018-12-01","ids":{"openalex":"https://openalex.org/W2915653915","doi":"https://doi.org/10.1109/padsw.2018.8644650","mag":"2915653915"},"language":"en","primary_location":{"id":"doi:10.1109/padsw.2018.8644650","is_oa":false,"landing_page_url":"https://doi.org/10.1109/padsw.2018.8644650","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2018 IEEE 24th International Conference on Parallel and Distributed Systems (ICPADS)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5062501723","display_name":"Fengfeng Pan","orcid":"https://orcid.org/0000-0002-6125-7681"},"institutions":[{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]},{"id":"https://openalex.org/I4391767888","display_name":"State Key Laboratory of Computer Architecture","ror":"https://ror.org/02pq9w205","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176","https://openalex.org/I4391767888"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fengfeng Pan","raw_affiliation_strings":["Institute of Computing Technology CAS, State Key Laboratory of Computer Architecture"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computing Technology CAS, State Key Laboratory of Computer Architecture","institution_ids":["https://openalex.org/I4210090176","https://openalex.org/I4391767888"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102854941","display_name":"Jin Xiong","orcid":"https://orcid.org/0000-0001-6025-9063"},"institutions":[{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]},{"id":"https://openalex.org/I4391767888","display_name":"State Key Laboratory of Computer Architecture","ror":"https://ror.org/02pq9w205","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176","https://openalex.org/I4391767888"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jin Xiong","raw_affiliation_strings":["Institute of Computing Technology CAS, State Key Laboratory of Computer Architecture"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computing Technology CAS, State Key Laboratory of Computer Architecture","institution_ids":["https://openalex.org/I4210090176","https://openalex.org/I4391767888"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017014737","display_name":"Yijie Shen","orcid":"https://orcid.org/0000-0002-2679-1088"},"institutions":[{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]},{"id":"https://openalex.org/I4391767888","display_name":"State Key Laboratory of Computer Architecture","ror":"https://ror.org/02pq9w205","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176","https://openalex.org/I4391767888"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yijie Shen","raw_affiliation_strings":["Institute of Computing Technology CAS, State Key Laboratory of Computer Architecture"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computing Technology CAS, State Key Laboratory of Computer Architecture","institution_ids":["https://openalex.org/I4210090176","https://openalex.org/I4391767888"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014809128","display_name":"Tianshi Wang","orcid":"https://orcid.org/0000-0002-8013-5188"},"institutions":[{"id":"https://openalex.org/I24185976","display_name":"Sichuan University","ror":"https://ror.org/011ashp19","country_code":"CN","type":"education","lineage":["https://openalex.org/I24185976"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianshi Wang","raw_affiliation_strings":["Sichuan University Chengdu China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sichuan University Chengdu China","institution_ids":["https://openalex.org/I24185976"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103223944","display_name":"Dejun Jiang","orcid":"https://orcid.org/0009-0001-0041-5957"},"institutions":[{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]},{"id":"https://openalex.org/I4391767888","display_name":"State Key Laboratory of Computer Architecture","ror":"https://ror.org/02pq9w205","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176","https://openalex.org/I4391767888"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dejun Jiang","raw_affiliation_strings":["Institute of Computing Technology CAS, State Key Laboratory of Computer Architecture"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computing Technology CAS, State Key Laboratory of Computer Architecture","institution_ids":["https://openalex.org/I4210090176","https://openalex.org/I4391767888"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8065130710601807},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.6069321632385254},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.5903775095939636},{"id":"https://openalex.org/keywords/storage-management","display_name":"Storage management","score":0.5065664052963257},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4974563419818878},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.45726004242897034},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.3626766502857208},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.05916619300842285}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8065130710601807},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.6069321632385254},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.5903775095939636},{"id":"https://openalex.org/C2984984529","wikidata":"https://www.wikidata.org/wiki/Q7619925","display_name":"Storage management","level":2,"score":0.5065664052963257},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4974563419818878},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.45726004242897034},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.3626766502857208},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.05916619300842285},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/padsw.2018.8644650","is_oa":false,"landing_page_url":"https://doi.org/10.1109/padsw.2018.8644650","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2018 IEEE 24th International Conference on Parallel and Distributed Systems (ICPADS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5600000023841858,"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W1514634512","https://openalex.org/W1572057369","https://openalex.org/W1579147384","https://openalex.org/W1861377444","https://openalex.org/W1985790218","https://openalex.org/W2023678166","https://openalex.org/W2041100566","https://openalex.org/W2052211129","https://openalex.org/W2080092502","https://openalex.org/W2094469165","https://openalex.org/W2096125134","https://openalex.org/W2101178023","https://openalex.org/W2110104287","https://openalex.org/W2110395205","https://openalex.org/W2131043098","https://openalex.org/W2140424185","https://openalex.org/W2140486418","https://openalex.org/W2150478767","https://openalex.org/W2155072926","https://openalex.org/W2381524109","https://openalex.org/W2406836379","https://openalex.org/W2522680599","https://openalex.org/W2543205889","https://openalex.org/W2787779818","https://openalex.org/W2798457757","https://openalex.org/W3104065274","https://openalex.org/W6630823317","https://openalex.org/W6634406473","https://openalex.org/W6634769544","https://openalex.org/W6639193275","https://openalex.org/W6676421412","https://openalex.org/W6679752225","https://openalex.org/W6710076213","https://openalex.org/W6713707562"],"related_works":["https://openalex.org/W3015859229","https://openalex.org/W2230552005","https://openalex.org/W1975949872","https://openalex.org/W3109411864","https://openalex.org/W3003280185","https://openalex.org/W3017846737","https://openalex.org/W4379407450","https://openalex.org/W2613379984","https://openalex.org/W2971569382","https://openalex.org/W2766461310"],"abstract_inverted_index":{"A":[0],"trend":[1],"in":[2,108,184],"nowadays":[3],"data":[4,21,54,68,73,97,154,174,209],"centers":[5],"is":[6,83,138],"that":[7,191],"heterogeneous":[8,77,128],"storage":[9,16,46,78,90,105,129,143,146,157,170,215],"devices":[10,91,106,114,144],"are":[11,27,98],"deployed":[12],"to":[13,41,43,139,176,200],"meet":[14],"different":[15,102,164,212],"demands":[17],"of":[18,66,89,104,111,136,142,163,166,214],"various":[19],"big":[20,53],"workloads.":[22],"For":[23],"example,":[24],"many":[25],"nodes":[26],"equipped":[28],"with":[29],"both":[30,153,169],"SSDs":[31],"and":[32,60,93,156,159,173,186,208],"HDDs.":[33],"And":[34],"HDFS":[35],"has":[36],"introduced":[37],"the":[38,64,72,87,96,101,150,161,187,205],"heterogeneous-storage-aware":[39],"feature":[40],"adapt":[42],"such":[44,115],"hybrid":[45],"clusters.":[47],"However,":[48],"current":[49],"task":[50,80,124],"scheduler":[51],"on":[52,204],"processing":[55],"platforms":[56],"(such":[57],"as":[58,116],"Hadoop":[59],"Spark)":[61],"only":[62],"considers":[63],"overhead":[65],"network":[67],"transmission":[69],"by":[70,86,145,152,168,198],"exploiting":[71],"locality":[74,155,175],"principle.":[75],"On":[76],"clusters,":[79],"completion":[81],"time":[82,197],"also":[84],"affected":[85],"speed":[88,103,113,172],"(SSDs":[92],"HDDs)":[94],"where":[95],"stored.":[99],"Ignoring":[100],"results":[107,189],"poor":[109],"utilization":[110],"high":[112],"SSD.":[117],"In":[118],"this":[119],"paper,":[120],"we":[121],"propose":[122],"a":[123],"scheduling":[125],"strategy":[126],"for":[127],"clusters":[130],"called":[131],"H-Scheduler.":[132],"The":[133],"key":[134],"idea":[135],"H-Scheduler":[137,183,192],"differentiate":[140],"speeds":[141],"types.":[147],"It":[148],"classifies":[149],"tasks":[151,167],"types,":[158],"redefines":[160],"priorities":[162],"classes":[165],"device":[171],"reduce":[177,194],"job":[178,195],"execution":[179,196],"time.":[180],"We":[181],"implemented":[182],"Spark,":[185],"experiment":[188],"show":[190],"can":[193],"up":[199],"73.6":[201],"%,":[202],"depending":[203],"workload":[206],"characteristics":[207],"distribution":[210],"among":[211],"types":[213],"devices.":[216]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":2},{"year":2019,"cited_by_count":3}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}