{"id":"https://openalex.org/W2899303430","doi":"https://doi.org/10.1109/tcc.2018.2878838","title":"An Intermediate Data Partition Algorithm for Skew Mitigation in Spark Computing Environment","display_name":"An Intermediate Data Partition Algorithm for Skew Mitigation in Spark Computing Environment","publication_year":2018,"publication_date":"2018-10-31","ids":{"openalex":"https://openalex.org/W2899303430","doi":"https://doi.org/10.1109/tcc.2018.2878838","mag":"2899303430"},"language":"en","primary_location":{"id":"doi:10.1109/tcc.2018.2878838","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcc.2018.2878838","pdf_url":null,"source":{"id":"https://openalex.org/S2492498579","display_name":"IEEE Transactions on Cloud Computing","issn_l":"2168-7161","issn":["2168-7161","2372-0018"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cloud Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008574161","display_name":"Zhuo Tang","orcid":"https://orcid.org/0000-0001-9081-8153"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhuo Tang","raw_affiliation_strings":["College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Changsha, Hunan, China"],"raw_orcid":"https://orcid.org/0000-0001-9081-8153","affiliations":[{"raw_affiliation_string":"College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Changsha, Hunan, China","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056242893","display_name":"Wei Lv","orcid":"https://orcid.org/0000-0003-0869-816X"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Lv","raw_affiliation_strings":["College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Changsha, Hunan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Changsha, Hunan, China","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078793726","display_name":"Kenli Li","orcid":"https://orcid.org/0000-0002-2635-7716"},"institutions":[{"id":"https://openalex.org/I157455823","display_name":"SUNY New Paltz","ror":"https://ror.org/03j3dv688","country_code":"US","type":"education","lineage":["https://openalex.org/I157455823"]},{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Kenli Li","raw_affiliation_strings":["College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Changsha, Hunan, China","Department of Computer Science, State University of New York, New Paltz, NY, USA"],"raw_orcid":"https://orcid.org/0000-0002-2635-7716","affiliations":[{"raw_affiliation_string":"College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Changsha, Hunan, China","institution_ids":["https://openalex.org/I16609230"]},{"raw_affiliation_string":"Department of Computer Science, State University of New York, New Paltz, NY, USA","institution_ids":["https://openalex.org/I157455823"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087894632","display_name":"Keqin Li","orcid":"https://orcid.org/0000-0001-5224-4048"},"institutions":[{"id":"https://openalex.org/I157455823","display_name":"SUNY New Paltz","ror":"https://ror.org/03j3dv688","country_code":"US","type":"education","lineage":["https://openalex.org/I157455823"]},{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Keqin Li","raw_affiliation_strings":["College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Changsha, Hunan, China","Department of Computer Science, State University of New York, New Paltz, NY, USA"],"raw_orcid":"https://orcid.org/0000-0001-5224-4048","affiliations":[{"raw_affiliation_string":"College of Information Science and Engineering, National Supercomputing Center in Changsha, Hunan University, Changsha, Hunan, China","institution_ids":["https://openalex.org/I16609230"]},{"raw_affiliation_string":"Department of Computer Science, State University of New York, New Paltz, NY, USA","institution_ids":["https://openalex.org/I157455823"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.0268,"has_fulltext":false,"cited_by_count":27,"citation_normalized_percentile":{"value":0.9483703,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"9","issue":"2","first_page":"461","last_page":"474"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10772","display_name":"Distributed systems and fault tolerance","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8077417612075806},{"id":"https://openalex.org/keywords/skew","display_name":"Skew","score":0.7892844676971436},{"id":"https://openalex.org/keywords/partition","display_name":"Partition (number theory)","score":0.7352977991104126},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.49714377522468567},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.4886772930622101},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4877379834651947},{"id":"https://openalex.org/keywords/sort","display_name":"sort","score":0.4872952699661255},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.48359817266464233},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.4688815772533417},{"id":"https://openalex.org/keywords/hash-function","display_name":"Hash function","score":0.4419100880622864},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.3426974415779114},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.22784143686294556},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.14681848883628845},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08739703893661499}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8077417612075806},{"id":"https://openalex.org/C43711488","wikidata":"https://www.wikidata.org/wiki/Q7534783","display_name":"Skew","level":2,"score":0.7892844676971436},{"id":"https://openalex.org/C42812","wikidata":"https://www.wikidata.org/wiki/Q1082910","display_name":"Partition (number theory)","level":2,"score":0.7352977991104126},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.49714377522468567},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.4886772930622101},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4877379834651947},{"id":"https://openalex.org/C88548561","wikidata":"https://www.wikidata.org/wiki/Q347599","display_name":"sort","level":2,"score":0.4872952699661255},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.48359817266464233},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.4688815772533417},{"id":"https://openalex.org/C99138194","wikidata":"https://www.wikidata.org/wiki/Q183427","display_name":"Hash function","level":2,"score":0.4419100880622864},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3426974415779114},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.22784143686294556},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.14681848883628845},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08739703893661499},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tcc.2018.2878838","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcc.2018.2878838","pdf_url":null,"source":{"id":"https://openalex.org/S2492498579","display_name":"IEEE Transactions on Cloud Computing","issn_l":"2168-7161","issn":["2168-7161","2372-0018"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cloud Computing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2833253907","display_name":null,"funder_award_id":"2017YFB0202201","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G3119699525","display_name":null,"funder_award_id":"L1624040","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3392145405","display_name":null,"funder_award_id":"L182400035","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5850822618","display_name":"\u503e\u659c\u4efb\u52a1\u8c03\u5ea6\u6a21\u578b\u53ca\u5176\u5728\u533b\u7597\u5927\u6570\u636e\u5904\u7406\u4e2d\u7684\u5e94\u7528\u7814\u7a76","funder_award_id":"61572176","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8752286753","display_name":"\u9ad8\u6027\u80fd\u5f02\u6784\u8ba1\u7b97\u73af\u5883\u4e2d\u7684\u6df1\u5ea6\u5b66\u4e60\u8fed\u4ee3\u4f18\u5316\u53ca\u5de5\u4e1a\u5e94\u7528\u7814\u7a76","funder_award_id":"61873090","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":37,"referenced_works":["https://openalex.org/W185167974","https://openalex.org/W1471136644","https://openalex.org/W1993667637","https://openalex.org/W2002722920","https://openalex.org/W2007333650","https://openalex.org/W2024077218","https://openalex.org/W2037027334","https://openalex.org/W2041958520","https://openalex.org/W2044936152","https://openalex.org/W2048455627","https://openalex.org/W2060650311","https://openalex.org/W2061807980","https://openalex.org/W2080131844","https://openalex.org/W2120755171","https://openalex.org/W2121562043","https://openalex.org/W2131975293","https://openalex.org/W2134952300","https://openalex.org/W2137373960","https://openalex.org/W2140509629","https://openalex.org/W2143108641","https://openalex.org/W2151487638","https://openalex.org/W2156441202","https://openalex.org/W2165769837","https://openalex.org/W2189465200","https://openalex.org/W2399133307","https://openalex.org/W2439404901","https://openalex.org/W2464834590","https://openalex.org/W2515208787","https://openalex.org/W2517850251","https://openalex.org/W2526832688","https://openalex.org/W2734430573","https://openalex.org/W6628579693","https://openalex.org/W6679815717","https://openalex.org/W6683267612","https://openalex.org/W6687322159","https://openalex.org/W6712438735","https://openalex.org/W6718333684"],"related_works":["https://openalex.org/W4290802965","https://openalex.org/W97789383","https://openalex.org/W3087516072","https://openalex.org/W2361805396","https://openalex.org/W1975949872","https://openalex.org/W3159871278","https://openalex.org/W2230552005","https://openalex.org/W4289406402","https://openalex.org/W4387423656","https://openalex.org/W4389136702"],"abstract_inverted_index":{"In":[0],"the":[1,21,34,42,64,68,77,81,85,99,104,114,117,144,152,160,172,223],"parallel":[2],"computing":[3],"framework":[4],"of":[5,20,44,71,80,108,116,162,226],"Hadoop/Spark,":[6],"data":[7,45,83,101,158,198],"skew":[8,66,145],"is":[9,33],"a":[10,54,94,216],"common":[11],"problem":[12],"resulting":[13],"in":[14,41,147,186],"performance":[15,192],"degradation,":[16],"such":[17],"as":[18],"prolonging":[19],"entire":[22,109],"execution":[23,224],"time":[24,225],"and":[25,57,84,130,142,171,189,203],"idle":[26],"resources.":[27],"What":[28],"lies":[29],"behind":[30],"this":[31],"issue":[32],"partition":[35,59,65,78,86,218],"imbalance,":[36],"that":[37,209],"causes":[38],"significant":[39,197],"differences":[40],"amount":[43],"processed":[46],"by":[47],"each":[48],"reduce":[49,222,227],"task.":[50],"This":[51],"paper":[52],"proposes":[53],"key":[55,106,126,133],"reassigning":[56,127],"splitting":[58,134],"algorithm":[60,96,128,135,211],"(SKRSP)":[61],"to":[62,102,113,155,178],"solve":[63],"from":[67],"source":[69],"codes":[70],"Spark-core_2.11":[72],"project,":[73],"which":[74,137],"considers":[75],"both":[76],"balance":[79,87,219],"intermediate":[82,110,157],"after":[88],"shuffle":[89,148],"operators.":[90],"First,":[91],"we":[92,120,183],"propose":[93],"step-based":[95],"for":[97,159,179],"sampling":[98],"input":[100],"estimate":[103],"general":[105],"distribution":[107],"data.":[111],"According":[112],"types":[115],"specific":[118],"applications,":[119],"design":[121],"two":[122],"algorithms:":[123],"hash":[124],"based":[125,132],"(KRHP)":[129],"rang":[131],"(KSRP),":[136],"can":[138,214],"generate":[139],"appropriate":[140],"strategy":[141],"implement":[143,184],"mitigation":[146],"phase.":[149],"KKSRP":[150],"generates":[151],"weighted":[153],"bounds":[154],"split":[156],"type":[161],"sort-based":[163],"applications":[164],"while":[165],"KRHP":[166],"records":[167],"these":[168,175],"reassigned":[169],"keys":[170,176],"new":[173],"reducers":[174],"belong":[177],"other":[180],"applications.":[181],"Finally,":[182],"SKRSP":[185],"Spark":[187],"2.2.0":[188],"evaluate":[190],"its":[191],"through":[193],"four":[194],"benchmarks":[195],"exhibiting":[196],"skew:":[199],"WordCount,":[200],"Sort,":[201],"Join,":[202],"PageRank.":[204],"The":[205],"experimental":[206],"results":[207],"verify":[208],"our":[210],"not":[212],"only":[213],"achieve":[215],"better":[217],"but":[220],"also":[221],"tasks":[228],"effectively.":[229]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}