{"id":"https://openalex.org/W2782998997","doi":"https://doi.org/10.1109/bigdata.2017.8257921","title":"Jointly optimizing task granularity and concurrency for in-memory mapreduce frameworks","display_name":"Jointly optimizing task granularity and concurrency for in-memory mapreduce frameworks","publication_year":2017,"publication_date":"2017-12-01","ids":{"openalex":"https://openalex.org/W2782998997","doi":"https://doi.org/10.1109/bigdata.2017.8257921","mag":"2782998997"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata.2017.8257921","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata.2017.8257921","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2017 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063201835","display_name":"Jonghyun Bae","orcid":"https://orcid.org/0000-0002-3987-6603"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jonghyun Bae","raw_affiliation_strings":["Department of Computer Science and Engineering, Seoul National University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Seoul National University","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063686935","display_name":"Hakbeom Jang","orcid":null},"institutions":[{"id":"https://openalex.org/I848706","display_name":"Sungkyunkwan University","ror":"https://ror.org/04q78tk20","country_code":"KR","type":"education","lineage":["https://openalex.org/I848706"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Hakbeom Jang","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Sungkyunkwan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Sungkyunkwan University","institution_ids":["https://openalex.org/I848706"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103244874","display_name":"Wenjing Jin","orcid":"https://orcid.org/0009-0008-1846-0458"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Wenjing Jin","raw_affiliation_strings":["Department of Computer Science and Engineering, Seoul National University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Seoul National University","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101780114","display_name":"Jun Heo","orcid":"https://orcid.org/0009-0007-7452-3030"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jun Heo","raw_affiliation_strings":["Department of Computer Science and Engineering, Seoul National University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Seoul National University","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041407656","display_name":"Jae-Young Jang","orcid":"https://orcid.org/0000-0002-3922-8542"},"institutions":[{"id":"https://openalex.org/I848706","display_name":"Sungkyunkwan University","ror":"https://ror.org/04q78tk20","country_code":"KR","type":"education","lineage":["https://openalex.org/I848706"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jaeyoung Jang","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Sungkyunkwan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Sungkyunkwan University","institution_ids":["https://openalex.org/I848706"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026955095","display_name":"Joo-Young Hwang","orcid":"https://orcid.org/0000-0001-5678-8499"},"institutions":[{"id":"https://openalex.org/I2250650973","display_name":"Samsung (South Korea)","ror":"https://ror.org/04w3jy968","country_code":"KR","type":"company","lineage":["https://openalex.org/I2250650973"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Joo-Young Hwang","raw_affiliation_strings":["Software Development Team, Memory Business, Samsung Electronics Co., Ltd"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Software Development Team, Memory Business, Samsung Electronics Co., Ltd","institution_ids":["https://openalex.org/I2250650973"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014181331","display_name":"Sangyeun Cho","orcid":null},"institutions":[{"id":"https://openalex.org/I2250650973","display_name":"Samsung (South Korea)","ror":"https://ror.org/04w3jy968","country_code":"KR","type":"company","lineage":["https://openalex.org/I2250650973"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Sangyeun Cho","raw_affiliation_strings":["Software Development Team, Memory Business, Samsung Electronics Co., Ltd"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Software Development Team, Memory Business, Samsung Electronics Co., Ltd","institution_ids":["https://openalex.org/I2250650973"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100415738","display_name":"Jae Wook Lee","orcid":"https://orcid.org/0000-0002-8756-0195"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jae W. Lee","raw_affiliation_strings":["Department of Computer Science and Engineering, Seoul National University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Seoul National University","institution_ids":["https://openalex.org/I139264467"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.0789,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.81447766,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"130","last_page":"140"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7802401781082153},{"id":"https://openalex.org/keywords/executor","display_name":"Executor","score":0.6262558698654175},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.6071550846099854},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5760387182235718},{"id":"https://openalex.org/keywords/memory-footprint","display_name":"Memory footprint","score":0.492180734872818},{"id":"https://openalex.org/keywords/tuple","display_name":"Tuple","score":0.4850303530693054},{"id":"https://openalex.org/keywords/granularity","display_name":"Granularity","score":0.4405498802661896},{"id":"https://openalex.org/keywords/garbage-collection","display_name":"Garbage collection","score":0.42907679080963135},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.4147932827472687},{"id":"https://openalex.org/keywords/garbage","display_name":"Garbage","score":0.2929845452308655},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.2367595136165619},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.22909918427467346},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08610457181930542}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7802401781082153},{"id":"https://openalex.org/C180591056","wikidata":"https://www.wikidata.org/wiki/Q654437","display_name":"Executor","level":2,"score":0.6262558698654175},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.6071550846099854},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5760387182235718},{"id":"https://openalex.org/C74912251","wikidata":"https://www.wikidata.org/wiki/Q6815727","display_name":"Memory footprint","level":2,"score":0.492180734872818},{"id":"https://openalex.org/C118930307","wikidata":"https://www.wikidata.org/wiki/Q600590","display_name":"Tuple","level":2,"score":0.4850303530693054},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.4405498802661896},{"id":"https://openalex.org/C105122174","wikidata":"https://www.wikidata.org/wiki/Q322202","display_name":"Garbage collection","level":3,"score":0.42907679080963135},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.4147932827472687},{"id":"https://openalex.org/C75403996","wikidata":"https://www.wikidata.org/wiki/Q5521979","display_name":"Garbage","level":2,"score":0.2929845452308655},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2367595136165619},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.22909918427467346},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08610457181930542},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata.2017.8257921","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata.2017.8257921","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2017 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W1503413821","https://openalex.org/W1746258828","https://openalex.org/W1788180225","https://openalex.org/W1975799560","https://openalex.org/W1976251241","https://openalex.org/W1992615224","https://openalex.org/W1994909161","https://openalex.org/W2040728701","https://openalex.org/W2131975293","https://openalex.org/W2150478767","https://openalex.org/W2155072926","https://openalex.org/W2166536280","https://openalex.org/W2173213060","https://openalex.org/W2206758307","https://openalex.org/W2269669481","https://openalex.org/W2475636809","https://openalex.org/W2496033776","https://openalex.org/W2511329048","https://openalex.org/W2557830883","https://openalex.org/W2559747646","https://openalex.org/W2570373436","https://openalex.org/W2615179483","https://openalex.org/W2963975954","https://openalex.org/W2998249817","https://openalex.org/W3104065274","https://openalex.org/W4233962317","https://openalex.org/W6637806892","https://openalex.org/W6638233953","https://openalex.org/W6679815717"],"related_works":["https://openalex.org/W2352852854","https://openalex.org/W2092389159","https://openalex.org/W4244773577","https://openalex.org/W4289493986","https://openalex.org/W2890045624","https://openalex.org/W2966867036","https://openalex.org/W2135804213","https://openalex.org/W1997217406","https://openalex.org/W2172070096","https://openalex.org/W2782998997"],"abstract_inverted_index":{"Recently,":[0],"in-memory":[1,23],"big":[2],"data":[3,19,83,206],"processing":[4],"frameworks":[5,26],"have":[6],"emerged,":[7],"such":[8],"as":[9,184],"Apache":[10],"Spark":[11,214,249],"and":[12,41,62,89,139,163,208,215,233,252,259],"Ignite,":[13],"to":[14,59,106,168,194,231,243],"accelerate":[15],"workloads":[16,220],"requiring":[17],"frequent":[18],"reuse.":[20],"With":[21],"effective":[22],"caching":[24],"these":[25,108],"eliminate":[27],"most":[28],"of":[29,82,91,100,122,159,172],"I/O":[30],"operations,":[31],"which":[32,141],"would":[33],"otherwise":[34],"be":[35],"necessary":[36],"for":[37,256],"communication":[38],"between":[39],"producer":[40],"consumer":[42],"tasks.":[43],"However,":[44],"this":[45,183],"performance":[46,228,255],"benefit":[47],"is":[48],"nullified":[49],"if":[50],"the":[51,69,98,123,156,186,235,246],"memory":[52,56],"footprint":[53],"exceeds":[54],"available":[55],"size,":[57],"due":[58],"excessive":[60],"spill":[61],"garbage":[63],"collection":[64],"(GC)":[65],"operations.":[66],"To":[67,148],"fit":[68],"working":[70],"set":[71],"in":[72,102,125],"memory,":[73],"two":[74],"system":[75],"parameters":[76,109,124,145],"play":[77],"an":[78,150,165,196],"important":[79],"role:":[80],"number":[81,90],"partitions":[84],"(Npartitions)":[85],"specifying":[86,97],"task":[87,137],"granularity,":[88],"tasks":[92],"per":[93],"each":[94,199],"executor":[95],"(Nthreads)":[96],"degree":[99],"parallelism":[101],"execution.":[103],"Existing":[104],"approaches":[105],"optimizing":[107],"either":[110],"do":[111],"not":[112],"take":[113],"into":[114],"account":[115],"workload":[116],"characteristics,":[117],"or":[118],"optimize":[119],"only":[120],"one":[121],"isolation,":[126],"thus":[127,201],"yielding":[128],"suboptimal":[129],"performance.":[130],"This":[131],"paper":[132],"introduces":[133],"WASP,":[134],"a":[135,160,190],"workload-aware":[136],"scheduler":[138,188],"partitioner,":[140],"jointly":[142],"optimizes":[143],"both":[144,257],"at":[146],"runtime.":[147],"find":[149,195],"optimal":[151,170,197],"setting,":[152],"WASP":[153,187,212,226],"first":[154],"analyzes":[155],"DAG":[157],"structure":[158],"given":[161],"workload,":[162],"uses":[164],"analytical":[166],"model":[167],"predict":[169],"settings":[171],"Npartitionsand":[173],"Nthreadsfor":[174,198],"all":[175],"stages":[176],"based":[177],"on":[178,213,221,239],"their":[179],"computation":[180],"types.":[181],"Taking":[182],"input,":[185],"employs":[189],"hill":[191],"climbing":[192],"algorithm":[193],"stage,":[200],"maximizing":[202],"concurrency":[203],"while":[204],"minimizing":[205],"spills":[207],"GCs.":[209],"We":[210],"prototype":[211],"evaluate":[216],"it":[217],"using":[218],"six":[219],"three":[222],"different":[223],"parallel":[224],"platforms.":[225],"improves":[227],"by":[229,241],"up":[230,242],"3.22\u00d7":[232],"reduces":[234],"cluster":[236],"operating":[237],"cost":[238],"cloud":[240],"40%,":[244],"over":[245],"baseline":[247],"following":[248],"Tuning":[250],"Guidelines":[251],"provides":[253],"robust":[254],"shuffle-heavy":[258],"shuffle-light":[260],"workloads.":[261]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}