{"id":"https://openalex.org/W2488548223","doi":"https://doi.org/10.1145/2907294.2907310","title":"Scaling Spark on HPC Systems","display_name":"Scaling Spark on HPC Systems","publication_year":2016,"publication_date":"2016-05-31","ids":{"openalex":"https://openalex.org/W2488548223","doi":"https://doi.org/10.1145/2907294.2907310","mag":"2488548223"},"language":"en","primary_location":{"id":"doi:10.1145/2907294.2907310","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2907294.2907310","pdf_url":null,"source":null,"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 25th ACM International Symposium on High-Performance Parallel and Distributed Computing","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://escholarship.org/content/qt84s551t7/qt84s551t7.pdf?t=oq1zye","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013775399","display_name":"Nicholas Chaimov","orcid":"https://orcid.org/0000-0001-7807-7620"},"institutions":[{"id":"https://openalex.org/I181233156","display_name":"University of Oregon","ror":"https://ror.org/0293rh119","country_code":"US","type":"education","lineage":["https://openalex.org/I181233156"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Nicholas Chaimov","raw_affiliation_strings":["University of Oregon, Eugene, OR, USA"],"affiliations":[{"raw_affiliation_string":"University of Oregon, Eugene, OR, USA","institution_ids":["https://openalex.org/I181233156"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029524538","display_name":"Allen D. Malony","orcid":"https://orcid.org/0000-0002-9598-7201"},"institutions":[{"id":"https://openalex.org/I181233156","display_name":"University of Oregon","ror":"https://ror.org/0293rh119","country_code":"US","type":"education","lineage":["https://openalex.org/I181233156"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Allen Malony","raw_affiliation_strings":["University of Oregon, Eugene, OR, USA"],"affiliations":[{"raw_affiliation_string":"University of Oregon, Eugene, OR, USA","institution_ids":["https://openalex.org/I181233156"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057883387","display_name":"Shane Canon","orcid":"https://orcid.org/0000-0002-8440-738X"},"institutions":[{"id":"https://openalex.org/I148283060","display_name":"Lawrence Berkeley National Laboratory","ror":"https://ror.org/02jbv0t02","country_code":"US","type":"facility","lineage":["https://openalex.org/I1330989302","https://openalex.org/I148283060","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shane Canon","raw_affiliation_strings":["Lawrence Berkeley National Laboratory, Berkeley, CA, USA"],"affiliations":[{"raw_affiliation_string":"Lawrence Berkeley National Laboratory, Berkeley, CA, USA","institution_ids":["https://openalex.org/I148283060"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023233400","display_name":"Costin Iancu","orcid":"https://orcid.org/0000-0001-7845-2427"},"institutions":[{"id":"https://openalex.org/I148283060","display_name":"Lawrence Berkeley National Laboratory","ror":"https://ror.org/02jbv0t02","country_code":"US","type":"facility","lineage":["https://openalex.org/I1330989302","https://openalex.org/I148283060","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Costin Iancu","raw_affiliation_strings":["Lawrence Berkeley National Laboratory, Berkeley, CA, USA"],"affiliations":[{"raw_affiliation_string":"Lawrence Berkeley National Laboratory, Berkeley, CA, USA","institution_ids":["https://openalex.org/I148283060"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062016419","display_name":"Khaled Z. Ibrahim","orcid":"https://orcid.org/0009-0004-5362-3612"},"institutions":[{"id":"https://openalex.org/I148283060","display_name":"Lawrence Berkeley National Laboratory","ror":"https://ror.org/02jbv0t02","country_code":"US","type":"facility","lineage":["https://openalex.org/I1330989302","https://openalex.org/I148283060","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Khaled Z. Ibrahim","raw_affiliation_strings":["Lawrence Berkeley National Laboratory, Berkeley, CA, USA"],"affiliations":[{"raw_affiliation_string":"Lawrence Berkeley National Laboratory, Berkeley, CA, USA","institution_ids":["https://openalex.org/I148283060"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081428368","display_name":"Jay Srinivasan","orcid":null},"institutions":[{"id":"https://openalex.org/I148283060","display_name":"Lawrence Berkeley National Laboratory","ror":"https://ror.org/02jbv0t02","country_code":"US","type":"facility","lineage":["https://openalex.org/I1330989302","https://openalex.org/I148283060","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jay Srinivasan","raw_affiliation_strings":["Lawrence Berkeley National Laboratory, Berkeley, CA, USA"],"affiliations":[{"raw_affiliation_string":"Lawrence Berkeley National Laboratory, Berkeley, CA, USA","institution_ids":["https://openalex.org/I148283060"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5013775399"],"corresponding_institution_ids":["https://openalex.org/I181233156"],"apc_list":null,"apc_paid":null,"fwci":15.9471,"has_fulltext":false,"cited_by_count":69,"citation_normalized_percentile":{"value":0.99226293,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"10","issue":null,"first_page":"97","last_page":"110"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8122344017028809},{"id":"https://openalex.org/keywords/lustre","display_name":"Lustre (file system)","score":0.7871785163879395},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.735190749168396},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.647998571395874},{"id":"https://openalex.org/keywords/file-system","display_name":"File system","score":0.6337229013442993},{"id":"https://openalex.org/keywords/porting","display_name":"Porting","score":0.5151035189628601},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.46376654505729675},{"id":"https://openalex.org/keywords/workstation","display_name":"Workstation","score":0.4619711935520172},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.3506578207015991}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8122344017028809},{"id":"https://openalex.org/C180699724","wikidata":"https://www.wikidata.org/wiki/Q1877782","display_name":"Lustre (file system)","level":3,"score":0.7871785163879395},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.735190749168396},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.647998571395874},{"id":"https://openalex.org/C2780940931","wikidata":"https://www.wikidata.org/wiki/Q174989","display_name":"File system","level":2,"score":0.6337229013442993},{"id":"https://openalex.org/C106251023","wikidata":"https://www.wikidata.org/wiki/Q851989","display_name":"Porting","level":3,"score":0.5151035189628601},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.46376654505729675},{"id":"https://openalex.org/C67953723","wikidata":"https://www.wikidata.org/wiki/Q192525","display_name":"Workstation","level":2,"score":0.4619711935520172},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.3506578207015991}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/2907294.2907310","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2907294.2907310","pdf_url":null,"source":null,"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 25th ACM International Symposium on High-Performance Parallel and Distributed Computing","raw_type":"proceedings-article"},{"id":"pmh:qt84s551t7","is_oa":true,"landing_page_url":"http://www.escholarship.org/uc/item/84s551t7","pdf_url":"https://escholarship.org/content/qt84s551t7/qt84s551t7.pdf?t=oq1zye","source":{"id":"https://openalex.org/S4306400115","display_name":"eScholarship (California Digital Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2801248553","host_organization_name":"California Digital Library","host_organization_lineage":["https://openalex.org/I2801248553"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Chaimov, N; Malony, A; Canon, S; Iancu, C; Ibrahim, KZ; & Srinivasan, J. (2016). Scaling spark on HPC systems. HPDC 2016 - Proceedings of the 25th ACM International Symposium on High-Performance Parallel and Distributed Computing, 97 - 110. doi: 10.1145/2907294.2907310. Lawrence Berkeley National Laboratory: Retrieved from: http://www.escholarship.org/uc/item/84s551t7","raw_type":"article"},{"id":"pmh:oai:escholarship.org/ark:/13030/qt84s551t7","is_oa":false,"landing_page_url":"https://escholarship.org/uc/item/84s551t7","pdf_url":null,"source":{"id":"https://openalex.org/S4306400115","display_name":"eScholarship (California Digital Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2801248553","host_organization_name":"California Digital Library","host_organization_lineage":["https://openalex.org/I2801248553"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"article"}],"best_oa_location":{"id":"pmh:qt84s551t7","is_oa":true,"landing_page_url":"http://www.escholarship.org/uc/item/84s551t7","pdf_url":"https://escholarship.org/content/qt84s551t7/qt84s551t7.pdf?t=oq1zye","source":{"id":"https://openalex.org/S4306400115","display_name":"eScholarship (California Digital Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2801248553","host_organization_name":"California Digital Library","host_organization_lineage":["https://openalex.org/I2801248553"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Chaimov, N; Malony, A; Canon, S; Iancu, C; Ibrahim, KZ; & Srinivasan, J. (2016). Scaling spark on HPC systems. HPDC 2016 - Proceedings of the 25th ACM International Symposium on High-Performance Parallel and Distributed Computing, 97 - 110. doi: 10.1145/2907294.2907310. Lawrence Berkeley National Laboratory: Retrieved from: http://www.escholarship.org/uc/item/84s551t7","raw_type":"article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6098739558","display_name":null,"funder_award_id":"DE-AC02-05CH11231","funder_id":"https://openalex.org/F4320310910","funder_display_name":"Savannah River Operations Office, U.S. Department of Energy"}],"funders":[{"id":"https://openalex.org/F4320310910","display_name":"Savannah River Operations Office, U.S. Department of Energy","ror":"https://ror.org/05hhm9a98"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W2488548223.pdf"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W410850256","https://openalex.org/W1448681276","https://openalex.org/W1598064945","https://openalex.org/W1746258828","https://openalex.org/W2035970724","https://openalex.org/W2038157364","https://openalex.org/W2038412523","https://openalex.org/W2039546699","https://openalex.org/W2067051372","https://openalex.org/W2071704971","https://openalex.org/W2089098642","https://openalex.org/W2123083497","https://openalex.org/W2124256298","https://openalex.org/W2131975293","https://openalex.org/W2144174917","https://openalex.org/W2163961697","https://openalex.org/W2173213060","https://openalex.org/W2189465200","https://openalex.org/W2346508825","https://openalex.org/W2739789142","https://openalex.org/W6687322159"],"related_works":["https://openalex.org/W4240440393","https://openalex.org/W2133280212","https://openalex.org/W4311545198","https://openalex.org/W2113098103","https://openalex.org/W2567604634","https://openalex.org/W2140836923","https://openalex.org/W3097895792","https://openalex.org/W2994238052","https://openalex.org/W2151392458","https://openalex.org/W2768462553"],"abstract_inverted_index":{"We":[0,57],"report":[1],"our":[2,27,125,150,159],"experiences":[3],"porting":[4],"Spark":[5,12],"to":[6,50,68,85,91,133,156,165],"large":[7,103],"production":[8],"HPC":[9,40,138],"systems.":[10],"While":[11],"performance":[13,48,89],"in":[14,38,136,170],"a":[15,39,54,59,80,99,102,137],"data":[16],"center":[17],"installation":[18,41,139],"(with":[19],"local":[20],"disks)":[21],"is":[22,130,163],"dominated":[23],"by":[24],"the":[25,75,94,110,119,171],"network,":[26],"results":[28,126],"show":[29],"that":[30,128],"file":[31,81,113],"system":[32,100],"metadata":[33],"access":[34],"latency":[35],"can":[36,153],"dominate":[37],"using":[42],"Lustre:":[43],"it":[44,162],"determines":[45],"single":[46],"node":[47,88],"up":[49,90,155],"4x":[51],"slower":[52],"than":[53],"typical":[55],"workstation.":[56],"evaluate":[58,98],"combination":[60],"of":[61,121],"software":[62,76],"techniques":[63],"and":[64,109,142],"hardware":[65,95],"configurations":[66],"designed":[67],"address":[69],"this":[70,115],"problem.":[71],"For":[72],"example,":[73],"on":[74],"side":[77,96],"we":[78,97,152],"develop":[79],"pooling":[82,151],"layer":[83],"able":[84],"improve":[86],"per":[87],"2.8x.":[92],"On":[93],"with":[101,140,149],"NVRAM":[104],"buffer":[105],"between":[106],"compute":[107],"nodes":[108],"backend":[111],"Lustre":[112,141],"system:":[114],"improves":[116],"scaling":[117],"at":[118],"expense":[120],"per-node":[122],"performance.":[123],"Overall,":[124],"indicate":[127],"scalability":[129,169],"currently":[131],"limited":[132],"O(102)":[134],"cores":[135],"default":[143],"Spark.":[144],"After":[145],"careful":[146],"configuration":[147],"combined":[148],"scale":[154],"O(10^4).":[157],"As":[158],"analysis":[160],"indicates,":[161],"feasible":[164],"observe":[166],"much":[167],"higher":[168],"near":[172],"future.":[173]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":6},{"year":2019,"cited_by_count":15},{"year":2018,"cited_by_count":17},{"year":2017,"cited_by_count":16},{"year":2016,"cited_by_count":6}],"updated_date":"2026-04-20T07:46:08.049788","created_date":"2025-10-10T00:00:00"}