{"id":"https://openalex.org/W2509495257","doi":"https://doi.org/10.5220/0005952300330043","title":"Simulating Spark Cluster for Deployment Planning, Evaluation and Optimization","display_name":"Simulating Spark Cluster for Deployment Planning, Evaluation and Optimization","publication_year":2016,"publication_date":"2016-01-01","ids":{"openalex":"https://openalex.org/W2509495257","doi":"https://doi.org/10.5220/0005952300330043","mag":"2509495257"},"language":"en","primary_location":{"id":"doi:10.5220/0005952300330043","is_oa":false,"landing_page_url":"https://doi.org/10.5220/0005952300330043","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 6th International Conference on Simulation and Modeling Methodologies, Technologies and Applications","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5084226477","display_name":"Qian Chen","orcid":"https://orcid.org/0000-0002-1841-6325"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Qian Chen","raw_affiliation_strings":["Intel Corporation, China"],"affiliations":[{"raw_affiliation_string":"Intel Corporation, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039120969","display_name":"Kebing Wang","orcid":"https://orcid.org/0009-0009-8212-769X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kebing Wang","raw_affiliation_strings":["Intel Corporation, China"],"affiliations":[{"raw_affiliation_string":"Intel Corporation, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090246112","display_name":"Zhaojuan Bian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhaojuan Bian","raw_affiliation_strings":["Intel Corporation, China"],"affiliations":[{"raw_affiliation_string":"Intel Corporation, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089316266","display_name":"Illia Cremer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Illia Cremer","raw_affiliation_strings":["Intel Corporation, France"],"affiliations":[{"raw_affiliation_string":"Intel Corporation, France","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022099164","display_name":"Gen Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gen Xu","raw_affiliation_strings":["Intel Corporation, China"],"affiliations":[{"raw_affiliation_string":"Intel Corporation, China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5010063789","display_name":"Yejun Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yejun Guo","raw_affiliation_strings":["Intel Corporation, China"],"affiliations":[{"raw_affiliation_string":"Intel Corporation, China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5084226477"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.10450965,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"33","last_page":"43"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.9775999784469604,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9589999914169312,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.8318760395050049},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8174511194229126},{"id":"https://openalex.org/keywords/provisioning","display_name":"Provisioning","score":0.5278853178024292},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.5223554372787476},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5097238421440125},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.4754980504512787},{"id":"https://openalex.org/keywords/computer-cluster","display_name":"Computer cluster","score":0.4342080056667328},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.3645660877227783},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.33976835012435913},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.33271849155426025}],"concepts":[{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.8318760395050049},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8174511194229126},{"id":"https://openalex.org/C172191483","wikidata":"https://www.wikidata.org/wiki/Q1071806","display_name":"Provisioning","level":2,"score":0.5278853178024292},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.5223554372787476},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5097238421440125},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.4754980504512787},{"id":"https://openalex.org/C29140674","wikidata":"https://www.wikidata.org/wiki/Q206637","display_name":"Computer cluster","level":2,"score":0.4342080056667328},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3645660877227783},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.33976835012435913},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.33271849155426025},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.5220/0005952300330043","is_oa":false,"landing_page_url":"https://doi.org/10.5220/0005952300330043","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 6th International Conference on Simulation and Modeling Methodologies, Technologies and Applications","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Responsible consumption and production","score":0.5299999713897705,"id":"https://metadata.un.org/sdg/12"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2766461310","https://openalex.org/W4247566972","https://openalex.org/W2941957272","https://openalex.org/W4388692845","https://openalex.org/W3202731209","https://openalex.org/W3211874991","https://openalex.org/W4246264554","https://openalex.org/W3112375919","https://openalex.org/W2962431048","https://openalex.org/W4233648438"],"abstract_inverted_index":{"As":[0],"the":[1,6,164,262],"most":[2],"active":[3],"project":[4],"in":[5,49],"Hadoop":[7,47],"ecosystem":[8],"these":[9,182],"days":[10],"(Zaharia,":[11],"2014),":[12,132],"Spark":[13,39,59,83,124,143,156,224,236],"is":[14,61,161,256],"a":[15,122,135,228],"fast":[16],"and":[17,35,68,73,80,94,100,108,116,149,175,194,227],"general":[18],"purpose":[19],"engine":[20,34],"for":[21,82,142,191],"large-scale":[22],"data":[23],"processing.":[24],"Thanks":[25],"to":[26,43],"its":[27],"advanced":[28],"Directed":[29],"Acyclic":[30],"Graph":[31],"(DAG)":[32],"execution":[33,158,268],"in-memory":[36],"computing":[37,251],"mechanism,":[38],"runs":[40,260],"programs":[41],"up":[42],"100x":[44],"faster":[45,53],"than":[46,266],"MapReduce":[48],"memory,":[50],"or":[51],"10x":[52],"on":[54,98,127,269],"disk":[55],"(Apache,":[56],"2016).":[57],"However,":[58],"performance":[60,147],"impacted":[62],"by":[63,163],"many":[64],"system":[65,152,202],"software,":[66],"hardware":[67,117],"dataset":[69],"factors":[70],"especially":[71,110],"memory":[72,138,201],"JVM":[74],"related,":[75],"which":[76],"makes":[77],"capacity":[78],"planning":[79,88],"tuning":[81],"clusters":[84],"extremely":[85],"difficult.":[86],"Current":[87],"methods":[89],"are":[90,95,104,185,210],"mostly":[91],"estimation":[92],"based":[93,126],"highly":[96],"dependent":[97],"experience":[99],"trial-and-error.":[101],"These":[102],"approaches":[103],"far":[105],"from":[106,181],"efficient":[107],"accurate,":[109],"with":[111,134,221,241],"increasing":[112],"software":[113,183],"stack":[114],"complexity":[115],"diversity.":[118],"Here,":[119],"we":[120],"propose":[121],"novel":[123],"simulator":[125,259],"CSMethod":[128],"(Bian":[129],"et":[130],"al.,":[131],"extension":[133],"fine-grained":[136,215],"multi-layered":[137],"subsystem,":[139],"well":[140],"suitable":[141],"cluster":[144,200],"deployment":[145],"planning,":[146],"evaluation":[148],"optimization":[150],"before":[151],"provisioning.":[153],"The":[154],"whole":[155],"application":[157],"life":[159],"cycle":[160],"simulated":[162],"proposed":[165],"simulator,":[166],"including":[167],"DAG":[168],"generation,":[169],"Resilient":[170],"Distributed":[171],"Dataset":[172],"(RDD)":[173],"processing":[174],"block":[176],"management.":[177],"Hardware":[178],"activities":[179],"derived":[180],"operations":[184],"dynamically":[186],"mapped":[187],"onto":[188],"architecture":[189],"models":[190],"processors,":[192],"storage,":[193],"network":[195],"devices.":[196],"Performance":[197],"behaviour":[198],"of":[199],"at":[203,261],"multiple":[204],"layers":[205],"(Spark,":[206],"JVM,":[207],"OS,":[208],"hardware)":[209],"modeled":[211],"as":[212],"an":[213,242],"enhanced":[214],"individual":[216],"global":[217],"library.":[218],"Experimental":[219],"results":[220],"several":[222],"popular":[223],"micro":[225],"benchmarks":[226],"real":[229],"case":[230],"IoT":[231],"workloads":[232],"demonstrate":[233],"that":[234],"our":[235,258],"Simulator":[237],"achieves":[238],"high":[239],"accuracy":[240],"average":[243],"error":[244],"rate":[245],"below":[246],"7%.":[247],"With":[248],"light":[249],"weight":[250],"resource":[252],"requirement":[253],"(a":[254],"laptop":[255],"enough)":[257],"same":[263],"speed":[264],"level":[265],"native":[267],"multi-node":[270],"high-end":[271],"cluster.":[272]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2020,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}