{"id":"https://openalex.org/W2396433687","doi":"https://doi.org/10.1109/tpds.2016.2587645","title":"iShuffle: Improving Hadoop Performance with Shuffle-on-Write","display_name":"iShuffle: Improving Hadoop Performance with Shuffle-on-Write","publication_year":2016,"publication_date":"2016-07-07","ids":{"openalex":"https://openalex.org/W2396433687","doi":"https://doi.org/10.1109/tpds.2016.2587645","mag":"2396433687"},"language":"en","primary_location":{"id":"doi:10.1109/tpds.2016.2587645","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2016.2587645","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024620270","display_name":"Yanfei Guo","orcid":"https://orcid.org/0000-0002-3731-5423"},"institutions":[{"id":"https://openalex.org/I888729015","display_name":"University of Colorado Colorado Springs","ror":"https://ror.org/054spjc55","country_code":"US","type":"education","lineage":["https://openalex.org/I888729015"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yanfei Guo","raw_affiliation_strings":["Department of Computer Science, University of Colorado, Colorado Springs, CO","University of Colorado Colorado Springs, Colorado Springs, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Colorado, Colorado Springs, CO","institution_ids":["https://openalex.org/I888729015"]},{"raw_affiliation_string":"University of Colorado Colorado Springs, Colorado Springs, United States","institution_ids":["https://openalex.org/I888729015"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073224556","display_name":"Jia Rao","orcid":"https://orcid.org/0000-0002-2133-4363"},"institutions":[{"id":"https://openalex.org/I888729015","display_name":"University of Colorado Colorado Springs","ror":"https://ror.org/054spjc55","country_code":"US","type":"education","lineage":["https://openalex.org/I888729015"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jia Rao","raw_affiliation_strings":["Department of Computer Science, University of Colorado, Colorado Springs, CO","University of Colorado Colorado Springs, Colorado Springs, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Colorado, Colorado Springs, CO","institution_ids":["https://openalex.org/I888729015"]},{"raw_affiliation_string":"University of Colorado Colorado Springs, Colorado Springs, United States","institution_ids":["https://openalex.org/I888729015"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Dazhao Cheng","orcid":null},"institutions":[{"id":"https://openalex.org/I888729015","display_name":"University of Colorado Colorado Springs","ror":"https://ror.org/054spjc55","country_code":"US","type":"education","lineage":["https://openalex.org/I888729015"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dazhao Cheng","raw_affiliation_strings":["Department of Computer Science, University of Colorado, Colorado Springs, CO","University of Colorado Colorado Springs, Colorado Springs, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Colorado, Colorado Springs, CO","institution_ids":["https://openalex.org/I888729015"]},{"raw_affiliation_string":"University of Colorado Colorado Springs, Colorado Springs, United States","institution_ids":["https://openalex.org/I888729015"]}]},{"author_position":"last","author":{"id":null,"display_name":"Xiaobo Zhou","orcid":null},"institutions":[{"id":"https://openalex.org/I888729015","display_name":"University of Colorado Colorado Springs","ror":"https://ror.org/054spjc55","country_code":"US","type":"education","lineage":["https://openalex.org/I888729015"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaobo Zhou","raw_affiliation_strings":["Department of Computer Science, University of Colorado, Colorado Springs, CO"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Colorado, Colorado Springs, CO","institution_ids":["https://openalex.org/I888729015"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I888729015"],"apc_list":{"value":2645,"currency":"USD","value_usd":2645},"apc_paid":null,"fwci":6.529,"has_fulltext":false,"cited_by_count":41,"citation_normalized_percentile":{"value":0.96995576,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"28","issue":"6","first_page":"1649","last_page":"1662"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12205","display_name":"Time Series Analysis and Forecasting","score":0.9868000149726868,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12205","display_name":"Time Series Analysis and Forecasting","score":0.9868000149726868,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10742","display_name":"Peer-to-Peer Network Technologies","score":0.9861000180244446,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11344","display_name":"Traffic Prediction and Management Techniques","score":0.9824000000953674,"subfield":{"id":"https://openalex.org/subfields/2215","display_name":"Building and Construction"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.814149022102356},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5533285140991211},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.47937920689582825},{"id":"https://openalex.org/keywords/storage-management","display_name":"Storage management","score":0.45380595326423645},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.3829255700111389}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.814149022102356},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5533285140991211},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.47937920689582825},{"id":"https://openalex.org/C2984984529","wikidata":"https://www.wikidata.org/wiki/Q7619925","display_name":"Storage management","level":2,"score":0.45380595326423645},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.3829255700111389}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpds.2016.2587645","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpds.2016.2587645","pdf_url":null,"source":{"id":"https://openalex.org/S97130795","display_name":"IEEE Transactions on Parallel and Distributed Systems","issn_l":"1045-9219","issn":["1045-9219","1558-2183","2161-9883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Parallel and Distributed Systems","raw_type":"journal-article"},{"id":"mag:2396433687","is_oa":false,"landing_page_url":"https://dblp.uni-trier.de/db/conf/icac/icac2013.html#GuoRZ13","pdf_url":null,"source":{"id":"https://openalex.org/S4306419164","display_name":"International Conference on Autonomic Computing","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":"International Conference on Autonomic Computing","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2033954303","display_name":"CSR: Small: System and Middleware Approaches to Predictable Services in Multi-Tenant Clouds","funder_award_id":"1320122","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2689826514","display_name":null,"funder_award_id":"61328203","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4187843670","display_name":"CAREER: Building Resilient Internet Services with Learning and Control","funder_award_id":"0844983","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4950096890","display_name":"CSR: Small: Moving MapReduce into the Cloud: Flexibility, Efficiency, and Elasticity","funder_award_id":"1422119","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7944385193","display_name":"CSR: Small: Autonomous Performance and Power Control on Virtualized Servers","funder_award_id":"1217979","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W167093911","https://openalex.org/W1508832460","https://openalex.org/W1997020216","https://openalex.org/W2002472616","https://openalex.org/W2013265062","https://openalex.org/W2041512590","https://openalex.org/W2067051372","https://openalex.org/W2076164405","https://openalex.org/W2081804145","https://openalex.org/W2094469165","https://openalex.org/W2096125134","https://openalex.org/W2104993419","https://openalex.org/W2105947650","https://openalex.org/W2107001003","https://openalex.org/W2112173609","https://openalex.org/W2118495646","https://openalex.org/W2122370965","https://openalex.org/W2140486418","https://openalex.org/W2140509629","https://openalex.org/W2141267666","https://openalex.org/W2150206209","https://openalex.org/W2173213060","https://openalex.org/W6628579693","https://openalex.org/W6639193275","https://openalex.org/W6639249596","https://openalex.org/W6677852368","https://openalex.org/W6679752225","https://openalex.org/W6690242045","https://openalex.org/W6712017604","https://openalex.org/W6758383543"],"related_works":["https://openalex.org/W2173213060","https://openalex.org/W1420267637","https://openalex.org/W1861377444","https://openalex.org/W2140509629","https://openalex.org/W2140486418","https://openalex.org/W2105947650","https://openalex.org/W2002472616","https://openalex.org/W2096125134","https://openalex.org/W2067051372","https://openalex.org/W2100830825","https://openalex.org/W2076164405","https://openalex.org/W167093911","https://openalex.org/W1508832460","https://openalex.org/W1471136644","https://openalex.org/W2189465200","https://openalex.org/W2114896543","https://openalex.org/W2110514750","https://openalex.org/W2077714512","https://openalex.org/W185167974","https://openalex.org/W1834532152"],"abstract_inverted_index":{"Hadoop":[0,46,77,89,93],"is":[1],"a":[2,87,133,141,154],"popular":[3],"implementation":[4],"of":[5,15,67,84,115],"the":[6,18,25,36,48,56,61,82,95,103],"MapReduce":[7],"framework":[8],"for":[9],"running":[10,55],"data-intensive":[11],"jobs":[12,85],"on":[13],"clusters":[14],"commodity":[16],"servers.Shuffle,":[17],"all-to-all":[19],"input":[20],"data":[21,104,150],"fetching":[22],"phase":[23,29,38,41],"between":[24,64],"map":[26,68,148],"and":[27,39,47,69,72,101,129,156,169,185,190],"reduce":[28,40,57,70,108,127,159],"can":[30,50],"significantly":[31,80],"affect":[32],"job":[33,116,177],"performance.":[34,117],"However,":[35],"shuffle":[37,49,125,143],"are":[42],"coupled":[43],"together":[44],"in":[45,75,86,188],"only":[51],"be":[52],"performed":[53],"by":[54,137,180],"tasks.":[58],"This":[59],"leaves":[60],"potential":[62],"parallelism":[63],"multiple":[65],"waves":[66],"unexploited":[71],"resource":[73],"wastage":[74],"multi-tenant":[76,88],"clusters,":[78,192],"which":[79,110],"delays":[81],"completion":[83,178],"cluster.":[90],"More":[91],"importantly,":[92],"lacks":[94],"ability":[96],"to":[97,112,123,151],"schedule":[98],"task":[99],"efficiently":[100],"mitigate":[102],"distribution":[105],"skew":[106],"among":[107],"tasks,":[109],"leads":[111],"further":[113],"degradation":[114],"In":[118],"this":[119],"work,":[120],"we":[121],"propose":[122],"decouple":[124],"from":[126],"tasks":[128,160],"convert":[130],"it":[131],"into":[132],"platform":[134],"service":[135,144],"provided":[136],"Hadoop.":[138],"We":[139],"presentiShuffle,":[140],"user-transparent":[142],"that":[145,174],"pro-actively":[146],"pushes":[147],"output":[149],"nodes":[152],"via":[153],"novelshuffle-on-writeoperation":[155],"flexibly":[157],"schedules":[158],"considering":[161],"workload":[162,171],"balance.":[163],"Experimental":[164],"results":[165],"with":[166],"representative":[167],"workloads":[168],"Facebook":[170],"trace":[172],"show":[173],"iShuffle":[175],"reduces":[176],"time":[179],"as":[181,183],"much":[182],"29.6":[184],"34":[186],"percent":[187],"single-user":[189],"multi-user":[191],"respectively.":[193]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":4},{"year":2017,"cited_by_count":11},{"year":2016,"cited_by_count":8},{"year":2015,"cited_by_count":8},{"year":2014,"cited_by_count":5},{"year":2013,"cited_by_count":1}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-10-10T00:00:00"}