{"id":"https://openalex.org/W2782998997","doi":"https://doi.org/10.1109/bigdata.2017.8257921","title":"Jointly optimizing task granularity and concurrency for in-memory mapreduce frameworks","display_name":"Jointly optimizing task granularity and concurrency for in-memory mapreduce frameworks","publication_year":2017,"publication_date":"2017-12-01","ids":{"openalex":"https://openalex.org/W2782998997","doi":"https://doi.org/10.1109/bigdata.2017.8257921","mag":"2782998997"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata.2017.8257921","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata.2017.8257921","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2017 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063201835","display_name":"Jonghyun Bae","orcid":"https://orcid.org/0000-0002-3987-6603"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":true,"raw_author_name":"Jonghyun Bae","raw_affiliation_strings":["Department of Computer Science and Engineering, Seoul National University"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Seoul National University","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063686935","display_name":"Hakbeom Jang","orcid":null},"institutions":[{"id":"https://openalex.org/I848706","display_name":"Sungkyunkwan University","ror":"https://ror.org/04q78tk20","country_code":"KR","type":"education","lineage":["https://openalex.org/I848706"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Hakbeom Jang","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Sungkyunkwan University"],"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Sungkyunkwan University","institution_ids":["https://openalex.org/I848706"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103244874","display_name":"Wenjing Jin","orcid":"https://orcid.org/0009-0008-1846-0458"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Wenjing Jin","raw_affiliation_strings":["Department of Computer Science and Engineering, Seoul National University"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Seoul National University","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101780114","display_name":"Jun Heo","orcid":"https://orcid.org/0009-0007-7452-3030"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jun Heo","raw_affiliation_strings":["Department of Computer Science and Engineering, Seoul National University"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Seoul National University","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041407656","display_name":"Jae-Young Jang","orcid":"https://orcid.org/0000-0002-3922-8542"},"institutions":[{"id":"https://openalex.org/I848706","display_name":"Sungkyunkwan University","ror":"https://ror.org/04q78tk20","country_code":"KR","type":"education","lineage":["https://openalex.org/I848706"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jaeyoung Jang","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Sungkyunkwan University"],"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Sungkyunkwan University","institution_ids":["https://openalex.org/I848706"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026955095","display_name":"Joo-Young Hwang","orcid":"https://orcid.org/0000-0001-5678-8499"},"institutions":[{"id":"https://openalex.org/I2250650973","display_name":"Samsung (South Korea)","ror":"https://ror.org/04w3jy968","country_code":"KR","type":"company","lineage":["https://openalex.org/I2250650973"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Joo-Young Hwang","raw_affiliation_strings":["Software Development Team, Memory Business, Samsung Electronics Co., Ltd"],"affiliations":[{"raw_affiliation_string":"Software Development Team, Memory Business, Samsung Electronics Co., Ltd","institution_ids":["https://openalex.org/I2250650973"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014181331","display_name":"Sangyeun Cho","orcid":null},"institutions":[{"id":"https://openalex.org/I2250650973","display_name":"Samsung (South Korea)","ror":"https://ror.org/04w3jy968","country_code":"KR","type":"company","lineage":["https://openalex.org/I2250650973"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Sangyeun Cho","raw_affiliation_strings":["Software Development Team, Memory Business, Samsung Electronics Co., Ltd"],"affiliations":[{"raw_affiliation_string":"Software Development Team, Memory Business, Samsung Electronics Co., Ltd","institution_ids":["https://openalex.org/I2250650973"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100415738","display_name":"Jae Wook Lee","orcid":"https://orcid.org/0000-0002-8756-0195"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jae W. Lee","raw_affiliation_strings":["Department of Computer Science and Engineering, Seoul National University"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Seoul National University","institution_ids":["https://openalex.org/I139264467"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5063201835"],"corresponding_institution_ids":["https://openalex.org/I139264467"],"apc_list":null,"apc_paid":null,"fwci":2.4181,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.91821191,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"130","last_page":"140"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7802401781082153},{"id":"https://openalex.org/keywords/executor","display_name":"Executor","score":0.6262558698654175},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.6071550846099854},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5760387182235718},{"id":"https://openalex.org/keywords/memory-footprint","display_name":"Memory footprint","score":0.492180734872818},{"id":"https://openalex.org/keywords/tuple","display_name":"Tuple","score":0.4850303530693054},{"id":"https://openalex.org/keywords/granularity","display_name":"Granularity","score":0.4405498802661896},{"id":"https://openalex.org/keywords/garbage-collection","display_name":"Garbage collection","score":0.42907679080963135},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.4147932827472687},{"id":"https://openalex.org/keywords/garbage","display_name":"Garbage","score":0.2929845452308655},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.2367595136165619},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.22909918427467346},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08610457181930542}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7802401781082153},{"id":"https://openalex.org/C180591056","wikidata":"https://www.wikidata.org/wiki/Q654437","display_name":"Executor","level":2,"score":0.6262558698654175},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.6071550846099854},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5760387182235718},{"id":"https://openalex.org/C74912251","wikidata":"https://www.wikidata.org/wiki/Q6815727","display_name":"Memory footprint","level":2,"score":0.492180734872818},{"id":"https://openalex.org/C118930307","wikidata":"https://www.wikidata.org/wiki/Q600590","display_name":"Tuple","level":2,"score":0.4850303530693054},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.4405498802661896},{"id":"https://openalex.org/C105122174","wikidata":"https://www.wikidata.org/wiki/Q322202","display_name":"Garbage collection","level":3,"score":0.42907679080963135},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.4147932827472687},{"id":"https://openalex.org/C75403996","wikidata":"https://www.wikidata.org/wiki/Q5521979","display_name":"Garbage","level":2,"score":0.2929845452308655},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2367595136165619},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.22909918427467346},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08610457181930542},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata.2017.8257921","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata.2017.8257921","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2017 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W1503413821","https://openalex.org/W1746258828","https://openalex.org/W1788180225","https://openalex.org/W1975799560","https://openalex.org/W1976251241","https://openalex.org/W1992615224","https://openalex.org/W1994909161","https://openalex.org/W2040728701","https://openalex.org/W2131975293","https://openalex.org/W2150478767","https://openalex.org/W2155072926","https://openalex.org/W2166536280","https://openalex.org/W2173213060","https://openalex.org/W2206758307","https://openalex.org/W2269669481","https://openalex.org/W2475636809","https://openalex.org/W2496033776","https://openalex.org/W2511329048","https://openalex.org/W2557830883","https://openalex.org/W2559747646","https://openalex.org/W2570373436","https://openalex.org/W2615179483","https://openalex.org/W2963975954","https://openalex.org/W2998249817","https://openalex.org/W3104065274","https://openalex.org/W4233962317","https://openalex.org/W6637806892","https://openalex.org/W6638233953","https://openalex.org/W6679815717"],"related_works":["https://openalex.org/W2352852854","https://openalex.org/W2092389159","https://openalex.org/W4244773577","https://openalex.org/W4289493986","https://openalex.org/W2890045624","https://openalex.org/W2966867036","https://openalex.org/W2135804213","https://openalex.org/W1997217406","https://openalex.org/W2172070096","https://openalex.org/W2782998997"],"abstract_inverted_index":{"Recently,":[0],"in-memory":[1,23],"big":[2],"data":[3,19,83,226],"processing":[4],"frameworks":[5,26],"have":[6],"emerged,":[7],"such":[8],"as":[9,200],"Apache":[10],"Spark":[11,234,269],"and":[12,41,62,93,147,171,185,228,235,253,272,279],"Ignite,":[13],"to":[14,59,114,176,210,251,263],"accelerate":[15],"workloads":[16,240],"requiring":[17],"frequent":[18],"reuse.":[20],"With":[21],"effective":[22],"caching":[24],"these":[25,116],"eliminate":[27],"most":[28],"of":[29,82,95,108,130,167,180],"I/O":[30],"operations,":[31],"which":[32,149],"would":[33],"otherwise":[34],"be":[35],"necessary":[36],"for":[37,190,218,276],"communication":[38],"between":[39],"producer":[40],"consumer":[42],"tasks.":[43],"However,":[44],"this":[45,199],"performance":[46,248,275],"benefit":[47],"is":[48],"nullified":[49],"if":[50],"the":[51,69,106,131,164,202,255,266],"memory":[52,56],"footprint":[53],"exceeds":[54],"available":[55],"size,":[57],"due":[58],"excessive":[60],"spill":[61],"garbage":[63],"collection":[64],"(GC)":[65],"operations.":[66],"To":[67,156],"fit":[68],"working":[70],"set":[71],"in":[72,110,133],"memory,":[73],"two":[74],"system":[75],"parameters":[76,117,132,153],"play":[77],"an":[78,158,173,212],"important":[79],"role:":[80],"number":[81,94],"partitions":[84],"(N":[85,100],"<inf":[86,101,182,187,215],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[87,102,183,188,216],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">partitions</inf>":[88,184],")":[89,104],"specifying":[90,105],"task":[91,145],"granularity,":[92],"tasks":[96],"per":[97],"each":[98,219],"executor":[99],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">threads</inf>":[103,189,217],"degree":[107],"parallelism":[109],"execution.":[111],"Existing":[112],"approaches":[113],"optimizing":[115],"either":[118],"do":[119],"not":[120],"take":[121],"into":[122],"account":[123],"workload":[124],"characteristics,":[125],"or":[126],"optimize":[127],"only":[128],"one":[129],"isolation,":[134],"thus":[135,221],"yielding":[136],"suboptimal":[137],"performance.":[138],"This":[139],"paper":[140],"introduces":[141],"WASP,":[142],"a":[143,168,206],"workload-aware":[144],"scheduler":[146,204],"partitioner,":[148],"jointly":[150],"optimizes":[151],"both":[152,277],"at":[154],"runtime.":[155],"find":[157,211],"optimal":[159,178,213],"setting,":[160],"WASP":[161,203,232,246],"first":[162],"analyzes":[163],"DAG":[165],"structure":[166],"given":[169],"workload,":[170],"uses":[172],"analytical":[174],"model":[175],"predict":[177],"settings":[179],"N":[181,186,214],"all":[191],"stages":[192],"based":[193],"on":[194,233,241,259],"their":[195],"computation":[196],"types.":[197],"Taking":[198],"input,":[201],"employs":[205],"hill":[207],"climbing":[208],"algorithm":[209],"stage,":[220],"maximizing":[222],"concurrency":[223],"while":[224],"minimizing":[225],"spills":[227],"GCs.":[229],"We":[230],"prototype":[231],"evaluate":[236],"it":[237],"using":[238],"six":[239],"three":[242],"different":[243],"parallel":[244],"platforms.":[245],"improves":[247],"by":[249,261],"up":[250,262],"3.22\u00d7":[252],"reduces":[254],"cluster":[256],"operating":[257],"cost":[258],"cloud":[260],"40%,":[264],"over":[265],"baseline":[267],"following":[268],"Tuning":[270],"Guidelines":[271],"provides":[273],"robust":[274],"shuffle-heavy":[278],"shuffle-light":[280],"workloads.":[281]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}