{"id":"https://openalex.org/W4388579568","doi":"https://doi.org/10.1109/tcc.2023.3329129","title":"Learning-Based Dynamic Memory Allocation Schemes for Apache Spark Data Processing","display_name":"Learning-Based Dynamic Memory Allocation Schemes for Apache Spark Data Processing","publication_year":2023,"publication_date":"2023-11-10","ids":{"openalex":"https://openalex.org/W4388579568","doi":"https://doi.org/10.1109/tcc.2023.3329129"},"language":"en","primary_location":{"id":"doi:10.1109/tcc.2023.3329129","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcc.2023.3329129","pdf_url":null,"source":{"id":"https://openalex.org/S2492498579","display_name":"IEEE Transactions on Cloud Computing","issn_l":"2168-7161","issn":["2168-7161","2372-0018"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cloud Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013649821","display_name":"Danlin Jia","orcid":"https://orcid.org/0000-0003-2858-0505"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Danlin Jia","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Northeastern University, Boston, MA, USA"],"raw_orcid":"https://orcid.org/0000-0003-2858-0505","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Northeastern University, Boston, MA, USA","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5118941484","display_name":"Li Wang","orcid":"https://orcid.org/0009-0006-0994-9564"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li Wang","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Northeastern University, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Northeastern University, Boston, MA, USA","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102578194","display_name":"Natalia Valencia","orcid":"https://orcid.org/0009-0002-6451-1830"},"institutions":[{"id":"https://openalex.org/I19700959","display_name":"Florida International University","ror":"https://ror.org/02gz6gg07","country_code":"US","type":"education","lineage":["https://openalex.org/I19700959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Natalia Valencia","raw_affiliation_strings":["School of Computing and Information Sciences, Florida International University, Miami, FL, USA"],"raw_orcid":"https://orcid.org/0009-0002-6451-1830","affiliations":[{"raw_affiliation_string":"School of Computing and Information Sciences, Florida International University, Miami, FL, USA","institution_ids":["https://openalex.org/I19700959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102777504","display_name":"Janki Bhimani","orcid":"https://orcid.org/0000-0002-4421-9923"},"institutions":[{"id":"https://openalex.org/I19700959","display_name":"Florida International University","ror":"https://ror.org/02gz6gg07","country_code":"US","type":"education","lineage":["https://openalex.org/I19700959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Janki Bhimani","raw_affiliation_strings":["School of Computing and Information Sciences, Florida International University, Miami, FL, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing and Information Sciences, Florida International University, Miami, FL, USA","institution_ids":["https://openalex.org/I19700959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085815921","display_name":"Bo Sheng","orcid":"https://orcid.org/0000-0001-5958-8935"},"institutions":[{"id":"https://openalex.org/I33434090","display_name":"University of Massachusetts Boston","ror":"https://ror.org/04ydmy275","country_code":"US","type":"education","lineage":["https://openalex.org/I33434090"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bo Sheng","raw_affiliation_strings":["Department of Computer Science, University of Massachusetts Boston, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Massachusetts Boston, Boston, MA, USA","institution_ids":["https://openalex.org/I33434090"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5091825452","display_name":"Ningfang Mi","orcid":"https://orcid.org/0009-0005-0934-1287"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ningfang Mi","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Northeastern University, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Northeastern University, Boston, MA, USA","institution_ids":["https://openalex.org/I12912129"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5013649821"],"corresponding_institution_ids":["https://openalex.org/I12912129"],"apc_list":null,"apc_paid":null,"fwci":0.4484,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.70926845,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":"12","issue":"1","first_page":"13","last_page":"25"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10270","display_name":"Blockchain Technology Applications and Security","score":0.9904000163078308,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8328630924224854},{"id":"https://openalex.org/keywords/garbage-collection","display_name":"Garbage collection","score":0.761233389377594},{"id":"https://openalex.org/keywords/memory-map","display_name":"Memory map","score":0.594842255115509},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.5760324597358704},{"id":"https://openalex.org/keywords/c-dynamic-memory-allocation","display_name":"C dynamic memory allocation","score":0.55621337890625},{"id":"https://openalex.org/keywords/interleaved-memory","display_name":"Interleaved memory","score":0.5515866875648499},{"id":"https://openalex.org/keywords/heap","display_name":"Heap (data structure)","score":0.5398023128509521},{"id":"https://openalex.org/keywords/memory-management","display_name":"Memory management","score":0.5251157879829407},{"id":"https://openalex.org/keywords/extended-memory","display_name":"Extended memory","score":0.5228265523910522},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.5223847031593323},{"id":"https://openalex.org/keywords/garbage","display_name":"Garbage","score":0.48877450823783875},{"id":"https://openalex.org/keywords/virtual-memory","display_name":"Virtual memory","score":0.45598557591438293},{"id":"https://openalex.org/keywords/flat-memory-model","display_name":"Flat memory model","score":0.4323938488960266},{"id":"https://openalex.org/keywords/registered-memory","display_name":"Registered memory","score":0.4282684326171875},{"id":"https://openalex.org/keywords/distributed-memory","display_name":"Distributed memory","score":0.42010581493377686},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.41189634799957275},{"id":"https://openalex.org/keywords/shared-memory","display_name":"Shared memory","score":0.34471678733825684},{"id":"https://openalex.org/keywords/overlay","display_name":"Overlay","score":0.2735040783882141},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.10504984855651855}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8328630924224854},{"id":"https://openalex.org/C105122174","wikidata":"https://www.wikidata.org/wiki/Q322202","display_name":"Garbage collection","level":3,"score":0.761233389377594},{"id":"https://openalex.org/C74426580","wikidata":"https://www.wikidata.org/wiki/Q719484","display_name":"Memory map","level":3,"score":0.594842255115509},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.5760324597358704},{"id":"https://openalex.org/C34339311","wikidata":"https://www.wikidata.org/wiki/Q1050390","display_name":"C dynamic memory allocation","level":4,"score":0.55621337890625},{"id":"https://openalex.org/C63511323","wikidata":"https://www.wikidata.org/wiki/Q908936","display_name":"Interleaved memory","level":4,"score":0.5515866875648499},{"id":"https://openalex.org/C134757568","wikidata":"https://www.wikidata.org/wiki/Q274089","display_name":"Heap (data structure)","level":2,"score":0.5398023128509521},{"id":"https://openalex.org/C176649486","wikidata":"https://www.wikidata.org/wiki/Q2308807","display_name":"Memory management","level":3,"score":0.5251157879829407},{"id":"https://openalex.org/C171675096","wikidata":"https://www.wikidata.org/wiki/Q1143380","display_name":"Extended memory","level":4,"score":0.5228265523910522},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.5223847031593323},{"id":"https://openalex.org/C75403996","wikidata":"https://www.wikidata.org/wiki/Q5521979","display_name":"Garbage","level":2,"score":0.48877450823783875},{"id":"https://openalex.org/C76399640","wikidata":"https://www.wikidata.org/wiki/Q189401","display_name":"Virtual memory","level":4,"score":0.45598557591438293},{"id":"https://openalex.org/C57863822","wikidata":"https://www.wikidata.org/wiki/Q905488","display_name":"Flat memory model","level":4,"score":0.4323938488960266},{"id":"https://openalex.org/C93446704","wikidata":"https://www.wikidata.org/wiki/Q449328","display_name":"Registered memory","level":3,"score":0.4282684326171875},{"id":"https://openalex.org/C91481028","wikidata":"https://www.wikidata.org/wiki/Q1054686","display_name":"Distributed memory","level":3,"score":0.42010581493377686},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.41189634799957275},{"id":"https://openalex.org/C133875982","wikidata":"https://www.wikidata.org/wiki/Q764810","display_name":"Shared memory","level":2,"score":0.34471678733825684},{"id":"https://openalex.org/C136085584","wikidata":"https://www.wikidata.org/wiki/Q910289","display_name":"Overlay","level":2,"score":0.2735040783882141},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.10504984855651855},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tcc.2023.3329129","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcc.2023.3329129","pdf_url":null,"source":{"id":"https://openalex.org/S2492498579","display_name":"IEEE Transactions on Cloud Computing","issn_l":"2168-7161","issn":["2168-7161","2372-0018"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cloud Computing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5899999737739563,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"awards":[{"id":"https://openalex.org/G1524590187","display_name":null,"funder_award_id":"CNS-1452751","funder_id":"https://openalex.org/F4320335353","funder_display_name":"National Science Foundation of Sri Lanka"},{"id":"https://openalex.org/G239424879","display_name":null,"funder_award_id":"CNS-2008072","funder_id":"https://openalex.org/F4320335353","funder_display_name":"National Science Foundation of Sri Lanka"},{"id":"https://openalex.org/G7393140921","display_name":null,"funder_award_id":"CNS-2323100","funder_id":"https://openalex.org/F4320335353","funder_display_name":"National Science Foundation of Sri Lanka"},{"id":"https://openalex.org/G8919562984","display_name":null,"funder_award_id":"CNS-2008324","funder_id":"https://openalex.org/F4320335353","funder_display_name":"National Science Foundation of Sri Lanka"}],"funders":[{"id":"https://openalex.org/F4320335353","display_name":"National Science Foundation of Sri Lanka","ror":"https://ror.org/010xaa060"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W1982003698","https://openalex.org/W2114296561","https://openalex.org/W2354203732","https://openalex.org/W2475738333","https://openalex.org/W2807562428","https://openalex.org/W2901280242","https://openalex.org/W2942295204","https://openalex.org/W2973184394","https://openalex.org/W2990209953","https://openalex.org/W2995564009","https://openalex.org/W3184147211","https://openalex.org/W4285503889","https://openalex.org/W6679815717","https://openalex.org/W6687322159","https://openalex.org/W6706155045"],"related_works":["https://openalex.org/W2753615087","https://openalex.org/W2296275612","https://openalex.org/W4248614727","https://openalex.org/W2491097902","https://openalex.org/W4243333834","https://openalex.org/W3048967625","https://openalex.org/W1554378476","https://openalex.org/W4293054943","https://openalex.org/W4214748026","https://openalex.org/W2074563599"],"abstract_inverted_index":{"Apache":[0],"Spark":[1,26,68,200,211,232,242],"is":[2,118],"an":[3,104,111],"in-memory":[4],"analytic":[5],"framework":[6],"that":[7,42,109,122,177,217],"has":[8],"been":[9],"adopted":[10],"in":[11,25,134,199,208,263],"the":[12,43,51,60,64,89,126,146,149,188,223,240,264],"industry":[13],"and":[14,21,36,95,151,186,202,228],"research":[15],"fields.":[16],"Two":[17],"memory":[18,29,45,53,80,86,93,106,153,174,197,219,243,251],"managers,":[19],"Static":[20],"Unified,":[22],"are":[23],"available":[24],"to":[27,83,142,183,239],"allocate":[28],"for":[30],"caching":[31],"Resilient":[32],"Distributed":[33],"Datasets":[34],"(RDDs)":[35],"executing":[37],"tasks.":[38],"However,":[39,116],"we":[40,74,167,255],"find":[41],"static":[44],"manager":[46,54,107,175,220,252],"(SMM)":[47],"lacks":[48],"flexibility,":[49],"while":[50],"unified":[52],"(UMM)":[55],"puts":[56],"heavy":[57],"pressure":[58],"on":[59,66],"garbage":[61,99,225],"collection":[62,226],"of":[63,91,128,159,190],"JVM":[65,160,191],"which":[67],"resides.":[69],"To":[70,163],"address":[71],"these":[72],"issues,":[73],"design":[75],"a":[76,119,135,156,170,179,209],"learning-based":[77,173],"bidirectional":[78],"usage-bounded":[79],"allocation":[81,87],"scheme":[82],"support":[84],"dynamic":[85],"with":[88],"consideration":[90],"both":[92],"demands":[94],"latency":[96],"introduced":[97],"by":[98,155,205],"collection.":[100],"We":[101,193],"first":[102],"develop":[103,169],"auto-tuning":[105],"(ATuMm)":[108],"adopts":[110],"intuitive":[112],"feedback-based":[113],"learning":[114],"solution.":[115],"ATuMm":[117,140],"slow":[120],"learner":[121],"can":[123,221,256],"only":[124],"alter":[125],"states":[127],"Java":[129],"Virtual":[130],"Memory":[131],"(JVM)":[132],"Heap":[133,161],"limited":[136],"range.":[137],"That":[138],"is,":[139],"decides":[141],"increase":[143],"or":[144],"decrease":[145],"boundary":[147],"between":[148],"execution":[150],"storage":[152],"pools":[154],"fixed":[157],"portion":[158],"size.":[162],"overcome":[164],"this":[165],"shortcoming,":[166],"further":[168,230,257],"new":[171,196],"reinforcement":[172],"(Q-ATuMm)":[176],"uses":[178],"Q-learning":[180],"intelligent":[181],"agent":[182],"dynamically":[184],"learn":[185],"tune":[187],"partition":[189],"Heap.":[192],"implement":[194],"our":[195,218,248],"managers":[198],"2.4.0":[201],"evaluate":[203],"them":[204],"conducting":[206],"experiments":[207],"real":[210],"cluster.":[212],"Our":[213],"experimental":[214],"results":[215],"show":[216],"reduce":[222],"total":[224],"time":[227],"thus":[229],"improve":[231],"applications\u2019":[233],"performance":[234],"(i.e.,":[235],"reduced":[236],"latency)":[237],"compared":[238],"existing":[241],"management":[244],"solutions.":[245],"By":[246],"integrating":[247],"machine":[249],"learning-driven":[250],"into":[253],"Spark,":[254],"obtain":[258],"around":[259],"1.3x":[260],"times":[261],"reduction":[262],"latency.":[265]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}