{"id":"https://openalex.org/W3048345275","doi":"https://doi.org/10.1145/3404397.3404430","title":"OPS: Optimized Shuffle Management System for Apache Spark","display_name":"OPS: Optimized Shuffle Management System for Apache Spark","publication_year":2020,"publication_date":"2020-08-09","ids":{"openalex":"https://openalex.org/W3048345275","doi":"https://doi.org/10.1145/3404397.3404430","mag":"3048345275"},"language":"en","primary_location":{"id":"doi:10.1145/3404397.3404430","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404397.3404430","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"49th International Conference on Parallel Processing - ICPP","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003318473","display_name":"Yuchen Cheng","orcid":"https://orcid.org/0000-0002-3230-2180"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuchen Cheng","raw_affiliation_strings":["Shanghai Jiao Tong University, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052741300","display_name":"Chunghsuan Wu","orcid":null},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chunghsuan Wu","raw_affiliation_strings":["Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101619452","display_name":"Yanqiang Liu","orcid":"https://orcid.org/0000-0002-8343-7923"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanqiang Liu","raw_affiliation_strings":["Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053323823","display_name":"Rui Ren","orcid":"https://orcid.org/0000-0003-3362-8083"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rui Ren","raw_affiliation_strings":["Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022420830","display_name":"Hong Xu","orcid":"https://orcid.org/0000-0002-9359-9571"},"institutions":[{"id":"https://openalex.org/I168719708","display_name":"City University of Hong Kong","ror":"https://ror.org/03q8dnn23","country_code":"HK","type":"education","lineage":["https://openalex.org/I168719708"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Hong Xu","raw_affiliation_strings":["City University of Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"City University of Hong Kong","institution_ids":["https://openalex.org/I168719708"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101893288","display_name":"Yang Bin","orcid":"https://orcid.org/0000-0002-8872-2985"},"institutions":[{"id":"https://openalex.org/I4210158342","display_name":"Intel (United Kingdom)","ror":"https://ror.org/058cxws58","country_code":"GB","type":"company","lineage":["https://openalex.org/I1343180700","https://openalex.org/I4210158342"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Bin Yang","raw_affiliation_strings":["Intel Corporation"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Intel Corporation","institution_ids":["https://openalex.org/I4210158342"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5011323970","display_name":"Zhengwei Qi","orcid":"https://orcid.org/0000-0003-2730-2319"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhengwei Qi","raw_affiliation_strings":["Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10270","display_name":"Blockchain Technology Applications and Security","score":0.9919999837875366,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7931237816810608},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.5657214522361755},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.5636137127876282},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.5586761832237244},{"id":"https://openalex.org/keywords/merge","display_name":"Merge (version control)","score":0.5423207879066467},{"id":"https://openalex.org/keywords/computer-cluster","display_name":"Computer cluster","score":0.4837789237499237},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.4835238456726074},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.4808686375617981},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.47766241431236267},{"id":"https://openalex.org/keywords/fault-tolerance","display_name":"Fault tolerance","score":0.43862950801849365},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.3822765648365021}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7931237816810608},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.5657214522361755},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.5636137127876282},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5586761832237244},{"id":"https://openalex.org/C197129107","wikidata":"https://www.wikidata.org/wiki/Q1921621","display_name":"Merge (version control)","level":2,"score":0.5423207879066467},{"id":"https://openalex.org/C29140674","wikidata":"https://www.wikidata.org/wiki/Q206637","display_name":"Computer cluster","level":2,"score":0.4837789237499237},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.4835238456726074},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.4808686375617981},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.47766241431236267},{"id":"https://openalex.org/C63540848","wikidata":"https://www.wikidata.org/wiki/Q3140932","display_name":"Fault tolerance","level":2,"score":0.43862950801849365},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.3822765648365021},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3404397.3404430","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404397.3404430","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"49th International Conference on Parallel Processing - ICPP","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.5799999833106995}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W1963832798","https://openalex.org/W2011039300","https://openalex.org/W2045271686","https://openalex.org/W2067051372","https://openalex.org/W2076164405","https://openalex.org/W2080131844","https://openalex.org/W2088687427","https://openalex.org/W2104680817","https://openalex.org/W2117702591","https://openalex.org/W2118566694","https://openalex.org/W2118676041","https://openalex.org/W2132353061","https://openalex.org/W2140509629","https://openalex.org/W2163404313","https://openalex.org/W2173213060","https://openalex.org/W2313564095","https://openalex.org/W2465838301","https://openalex.org/W2497069692","https://openalex.org/W2604258491","https://openalex.org/W2621239346","https://openalex.org/W2787779818","https://openalex.org/W2798457757","https://openalex.org/W4230059272","https://openalex.org/W4239666938","https://openalex.org/W4246762826"],"related_works":["https://openalex.org/W1862835629","https://openalex.org/W2099111379","https://openalex.org/W2136799148","https://openalex.org/W2897533804","https://openalex.org/W2890506991","https://openalex.org/W4246264554","https://openalex.org/W2962431048","https://openalex.org/W3112375919","https://openalex.org/W4233648438","https://openalex.org/W2969378954"],"abstract_inverted_index":{"In":[0,164],"recent":[1],"years,":[2],"distributed":[3,69],"computing":[4,70,106,139],"frameworks,":[5],"such":[6],"as":[7],"Hadoop":[8],"MapReduce":[9],"and":[10,46,88,100,105,117],"Spark,":[11,76],"are":[12,49],"widely":[13],"used":[14],"for":[15,83],"big":[16],"data":[17,33],"processing.":[18],"With":[19],"the":[20,24,35,53,58,61,93,97,103,119,124,135,143,157,165],"explosive":[21],"growth":[22],"of":[23,26,34,41,60,123,138,145,159,168],"amount":[25],"data,":[27],"companies":[28],"tend":[29],"to":[30,115,133],"store":[31],"intermediate":[32],"shuffle":[36,54,62,71,81,98,160],"phase":[37,99],"on":[38,75,147,178],"disk":[39,47],"instead":[40],"memory.":[42],"Therefore,":[43],"intensive":[44],"network":[45],"I/O":[48,94,104],"both":[50],"involved":[51],"in":[52,96],"phase.":[55],"To":[56],"optimize":[57],"overhead":[59,95,158],"phase,":[63],"we":[64],"propose":[65],"OPS,":[66],"an":[67,79],"open-source":[68],"management":[72],"system":[73],"based":[74],"which":[77],"provides":[78,129],"independent":[80],"service":[82],"Spark.":[84],"By":[85],"using":[86],"early-merge":[87],"early-shuffle":[89],"strategy,":[90],"OPS":[91,108,128,146,155,170],"alleviates":[92],"efficiently":[101],"schedules":[102],"resources.":[107],"also":[109],"proposes":[110],"a":[111,130,148],"slot-based":[112],"scheduling":[113,121],"algorithm":[114],"predict":[116],"calculate":[118],"optimal":[120],"result":[122],"reduce":[125],"task.":[126],"Besides,":[127],"taint-redo":[131],"strategy":[132],"ensure":[134],"fault":[136],"tolerance":[137],"jobs.":[140],"We":[141],"evaluated":[142],"performance":[144],"100-node":[149],"Amazon":[150],"AWS":[151],"EC2":[152],"cluster.":[153],"Overall,":[154],"optimizes":[156],"by":[161,175],"nearly":[162,176],"50%.":[163],"test":[166],"cases":[167],"HiBench,":[169],"improves":[171],"end-to-end":[172],"completion":[173],"time":[174],"30%":[177],"average.":[179]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}