{"id":"https://openalex.org/W2328834383","doi":"https://doi.org/10.1109/tc.2016.2532860","title":"Dynamic Resource Allocation for MapReduce with Partitioning Skew","display_name":"Dynamic Resource Allocation for MapReduce with Partitioning Skew","publication_year":2016,"publication_date":"2016-02-23","ids":{"openalex":"https://openalex.org/W2328834383","doi":"https://doi.org/10.1109/tc.2016.2532860","mag":"2328834383"},"language":"en","primary_location":{"id":"doi:10.1109/tc.2016.2532860","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tc.2016.2532860","pdf_url":null,"source":{"id":"https://openalex.org/S157670870","display_name":"IEEE Transactions on Computers","issn_l":"0018-9340","issn":["0018-9340","0016-9340","1557-9956","2326-3814"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Computers","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100328498","display_name":"Zhihong Liu","orcid":"https://orcid.org/0000-0002-2734-0246"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]},{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CA","CN"],"is_corresponding":false,"raw_author_name":"Zhihong Liu","raw_affiliation_strings":["College of Computer, National University of Defense Technology, Changsha, China","David R. Cheriton School of Computer Science, University of Waterloo, Waterloo, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Computer, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]},{"raw_affiliation_string":"David R. Cheriton School of Computer Science, University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100360194","display_name":"Qi Zhang","orcid":"https://orcid.org/0000-0001-5303-9804"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Qi Zhang","raw_affiliation_strings":["David R. Cheriton School of Computer Science, University of Waterloo"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"David R. Cheriton School of Computer Science, University of Waterloo","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103440577","display_name":"Reaz Ahmed","orcid":null},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Reaz Ahmed","raw_affiliation_strings":["David R. Cheriton School of Computer Science, University of Waterloo"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"David R. Cheriton School of Computer Science, University of Waterloo","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038723583","display_name":"Raouf Boutaba","orcid":"https://orcid.org/0000-0001-7936-6862"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Raouf Boutaba","raw_affiliation_strings":["David R. Cheriton School of Computer Science, University of Waterloo"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"David R. Cheriton School of Computer Science, University of Waterloo","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100761721","display_name":"Yaping Liu","orcid":"https://orcid.org/0000-0002-1076-9046"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yaping Liu","raw_affiliation_strings":["National University of Defense Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100295989","display_name":"Zhenghu Gong","orcid":null},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhenghu Gong","raw_affiliation_strings":["National University of Defense Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":10.1494,"has_fulltext":false,"cited_by_count":26,"citation_normalized_percentile":{"value":0.97933241,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"65","issue":"11","first_page":"3304","last_page":"3317"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.9843000173568726,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/skew","display_name":"Skew","score":0.9058361053466797},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7769336700439453},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.6823623180389404},{"id":"https://openalex.org/keywords/partition","display_name":"Partition (number theory)","score":0.6434515714645386},{"id":"https://openalex.org/keywords/notation","display_name":"Notation","score":0.6406848430633545},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.597091794013977},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.5300424098968506},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5024678707122803},{"id":"https://openalex.org/keywords/yarn","display_name":"Yarn","score":0.4739653766155243},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.4515414237976074},{"id":"https://openalex.org/keywords/node","display_name":"Node (physics)","score":0.43857526779174805},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4271208643913269},{"id":"https://openalex.org/keywords/factor","display_name":"Factor (programming language)","score":0.4125380218029022},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.153633713722229},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.11066135764122009},{"id":"https://openalex.org/keywords/arithmetic","display_name":"Arithmetic","score":0.08342954516410828}],"concepts":[{"id":"https://openalex.org/C43711488","wikidata":"https://www.wikidata.org/wiki/Q7534783","display_name":"Skew","level":2,"score":0.9058361053466797},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7769336700439453},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.6823623180389404},{"id":"https://openalex.org/C42812","wikidata":"https://www.wikidata.org/wiki/Q1082910","display_name":"Partition (number theory)","level":2,"score":0.6434515714645386},{"id":"https://openalex.org/C45357846","wikidata":"https://www.wikidata.org/wiki/Q2001982","display_name":"Notation","level":2,"score":0.6406848430633545},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.597091794013977},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5300424098968506},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5024678707122803},{"id":"https://openalex.org/C2778787235","wikidata":"https://www.wikidata.org/wiki/Q49007","display_name":"Yarn","level":2,"score":0.4739653766155243},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.4515414237976074},{"id":"https://openalex.org/C62611344","wikidata":"https://www.wikidata.org/wiki/Q1062658","display_name":"Node (physics)","level":2,"score":0.43857526779174805},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4271208643913269},{"id":"https://openalex.org/C2781039887","wikidata":"https://www.wikidata.org/wiki/Q1391724","display_name":"Factor (programming language)","level":2,"score":0.4125380218029022},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.153633713722229},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11066135764122009},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.08342954516410828},{"id":"https://openalex.org/C66938386","wikidata":"https://www.wikidata.org/wiki/Q633538","display_name":"Structural engineering","level":1,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tc.2016.2532860","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tc.2016.2532860","pdf_url":null,"source":{"id":"https://openalex.org/S157670870","display_name":"IEEE Transactions on Computers","issn_l":"0018-9340","issn":["0018-9340","0016-9340","1557-9956","2326-3814"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Computers","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.6399999856948853}],"awards":[{"id":"https://openalex.org/G7078575212","display_name":"\u62b5\u5fa1\u8fdd\u53cd\u8def\u7531\u7b56\u7565\u653b\u51fb\u7684\u53ef\u9a8c\u8bc1\u5b89\u5168\u8def\u7531\u534f\u8bae\u7684\u7814\u7a76","funder_award_id":"61472438","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8352776316","display_name":null,"funder_award_id":"NETGP394424-10","funder_id":"https://openalex.org/F4320334593","funder_display_name":"Natural Sciences and Engineering Research Council of Canada"}],"funders":[{"id":"https://openalex.org/F4320310630","display_name":"University of Ottawa","ror":"https://ror.org/03c4mmv16"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322015","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087"},{"id":"https://openalex.org/F4320322676","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68"},{"id":"https://openalex.org/F4320323059","display_name":"South China University of Technology","ror":"https://ror.org/0530pts50"},{"id":"https://openalex.org/F4320324150","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":44,"referenced_works":["https://openalex.org/W17864305","https://openalex.org/W1508832460","https://openalex.org/W1516820013","https://openalex.org/W1630733292","https://openalex.org/W1834532152","https://openalex.org/W1861377444","https://openalex.org/W1890643295","https://openalex.org/W2002472616","https://openalex.org/W2003138918","https://openalex.org/W2007333650","https://openalex.org/W2040141901","https://openalex.org/W2057279099","https://openalex.org/W2058047606","https://openalex.org/W2059348194","https://openalex.org/W2064956312","https://openalex.org/W2070162726","https://openalex.org/W2073292498","https://openalex.org/W2080131844","https://openalex.org/W2105947650","https://openalex.org/W2110104287","https://openalex.org/W2110514750","https://openalex.org/W2122370965","https://openalex.org/W2126399065","https://openalex.org/W2132549764","https://openalex.org/W2140509629","https://openalex.org/W2141563029","https://openalex.org/W2154042331","https://openalex.org/W2154399174","https://openalex.org/W2165769837","https://openalex.org/W2166418599","https://openalex.org/W2173213060","https://openalex.org/W2238447664","https://openalex.org/W2243163195","https://openalex.org/W4211012425","https://openalex.org/W4246315676","https://openalex.org/W4251459357","https://openalex.org/W6630886810","https://openalex.org/W6639007902","https://openalex.org/W6639193275","https://openalex.org/W6639249596","https://openalex.org/W6667993270","https://openalex.org/W6676421412","https://openalex.org/W6690021429","https://openalex.org/W6690829181"],"related_works":["https://openalex.org/W2078851640","https://openalex.org/W2381070915","https://openalex.org/W2352481835","https://openalex.org/W2392493391","https://openalex.org/W4200164335","https://openalex.org/W2109471562","https://openalex.org/W2090695246","https://openalex.org/W2378508949","https://openalex.org/W4387423656","https://openalex.org/W4389136702"],"abstract_inverted_index":{"MapReduce":[0,20],"has":[1],"become":[2],"a":[3,46,77,131,156,181],"prevalent":[4],"programming":[5],"model":[6],"for":[7],"building":[8],"data":[9],"processing":[10],"applications":[11],"in":[12],"the":[13,32,65,95,101,114,141,149,164,170,176],"cloud.":[14],"While":[15],"being":[16],"widely":[17],"used,":[18],"existing":[19],"schedulers":[21],"still":[22],"suffer":[23],"from":[24],"an":[25],"issue":[26],"known":[27],"as":[28],"partitioning":[29,82,96,145],"skew,":[30,146],"where":[31],"output":[33],"of":[34,86,103,144,158,183],"map":[35],"tasks":[36],"is":[37,118],"unevenly":[38],"distributed":[39],"among":[40,52,89],"reduce":[41,53,90,108],"tasks.":[42,54],"Existing":[43],"solutions":[44],"follow":[45],"similar":[47],"principle":[48],"that":[49,79,136],"repartitions":[50],"workload":[51,88],"However,":[55],"those":[56],"approaches":[57],"often":[58],"incur":[59],"high":[60],"performance":[61],"overhead":[62],"due":[63],"to":[64,106,120,155,169],"partition":[66],"size":[67],"prediction":[68],"and":[69,126],"repartitioning.":[70],"In":[71],"this":[72],"paper,":[73],"we":[74,92],"present":[75],"DREAMS,":[76],"framework":[78],"provides":[80],"run-time":[81],"skew":[83,97],"mitigation.":[84],"Instead":[85],"repartitioning":[87,115],"tasks,":[91],"cope":[93],"with":[94],"problem":[98],"by":[99,153,180],"controlling":[100],"amount":[102],"resources":[104],"allocated":[105],"each":[107],"task.":[109],"Our":[110],"approach":[111],"completely":[112],"eliminates":[113],"overhead,":[116],"yet":[117],"simple":[119],"implement.":[121],"Experiments":[122],"using":[123],"both":[124],"real":[125],"synthetic":[127],"workloads":[128],"running":[129],"on":[130],"21-node":[132],"Hadoop":[133,166],"cluster":[134],"demonstrate":[135],"DREAMS":[137,173],"can":[138,174],"effectively":[139],"mitigate":[140],"negative":[142],"impact":[143],"thereby":[147],"improving":[148],"job":[150,177],"completion":[151,178],"time":[152,179],"up":[154],"factor":[157,182],"<inline-formula><tex-math":[159,184],"notation=\"LaTeX\">":[160],"$2.29$</tex-math>":[161],"</inline-formula>":[162],"over":[163],"native":[165],"YARN.":[167],"Compared":[168],"state-of-the-art":[171],"solution,":[172],"improve":[175],"notation=\"LaTeX\">$1.65$</tex-math></inline-formula>":[185],".":[186]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":7},{"year":2018,"cited_by_count":4},{"year":2017,"cited_by_count":4}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}