{"id":"https://openalex.org/W2908586127","doi":"https://doi.org/10.1109/bdcat.2018.00010","title":"Spark-DIY: A Framework for Interoperable Spark Operations with High Performance Block-Based Data Models","display_name":"Spark-DIY: A Framework for Interoperable Spark Operations with High Performance Block-Based Data Models","publication_year":2018,"publication_date":"2018-12-01","ids":{"openalex":"https://openalex.org/W2908586127","doi":"https://doi.org/10.1109/bdcat.2018.00010","mag":"2908586127"},"language":"en","primary_location":{"id":"doi:10.1109/bdcat.2018.00010","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bdcat.2018.00010","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2018 IEEE/ACM 5th International Conference on Big Data Computing Applications and Technologies (BDCAT)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://e-archivo.uc3m.es/bitstreams/a288dee3-7cc2-4d69-8a3f-bec4a212bb3e/download","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052995286","display_name":"Silvina Ca\u00edno\u2010Lores","orcid":"https://orcid.org/0000-0002-6922-0138"},"institutions":[{"id":"https://openalex.org/I50357001","display_name":"Universidad Carlos III de Madrid","ror":"https://ror.org/03ths8210","country_code":"ES","type":"education","lineage":["https://openalex.org/I50357001"]}],"countries":["ES"],"is_corresponding":true,"raw_author_name":"Silvina Caino-Lores","raw_affiliation_strings":["Department of Computer Science and Engineering, University Carlos III of Madrid, Legan\u00e9s, Spain"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University Carlos III of Madrid, Legan\u00e9s, Spain","institution_ids":["https://openalex.org/I50357001"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068077636","display_name":"Jes\u00fas Carretero","orcid":"https://orcid.org/0000-0002-1413-4793"},"institutions":[{"id":"https://openalex.org/I50357001","display_name":"Universidad Carlos III de Madrid","ror":"https://ror.org/03ths8210","country_code":"ES","type":"education","lineage":["https://openalex.org/I50357001"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Jesus Carretero","raw_affiliation_strings":["Department of Computer Science and Engineering, University Carlos III of Madrid, Legan\u00e9s, Spain"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University Carlos III of Madrid, Legan\u00e9s, Spain","institution_ids":["https://openalex.org/I50357001"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085745891","display_name":"Bogdan Nicolae","orcid":"https://orcid.org/0000-0002-0661-7509"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bogdan Nicolae","raw_affiliation_strings":["Mathematics and Computer Science Division, Argonne National Laboratory, Lemont, IL, USA"],"affiliations":[{"raw_affiliation_string":"Mathematics and Computer Science Division, Argonne National Laboratory, Lemont, IL, USA","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106355290","display_name":"Or\u00e7un Yildiz","orcid":"https://orcid.org/0009-0006-5910-9221"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Orcun Yildiz","raw_affiliation_strings":["Mathematics and Computer Science Division, Argonne National Laboratory, Lemont, IL, USA"],"affiliations":[{"raw_affiliation_string":"Mathematics and Computer Science Division, Argonne National Laboratory, Lemont, IL, USA","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5108096533","display_name":"Tom Peterka","orcid":null},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tom Peterka","raw_affiliation_strings":["Mathematics and Computer Science Division, Argonne National Laboratory, Lemont, IL, USA"],"affiliations":[{"raw_affiliation_string":"Mathematics and Computer Science Division, Argonne National Laboratory, Lemont, IL, USA","institution_ids":["https://openalex.org/I1282105669"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5052995286"],"corresponding_institution_ids":["https://openalex.org/I50357001"],"apc_list":null,"apc_paid":null,"fwci":2.345,"has_fulltext":true,"cited_by_count":17,"citation_normalized_percentile":{"value":0.8995025,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8150862455368042},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.7391349077224731},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.6770385503768921},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6432515978813171},{"id":"https://openalex.org/keywords/programming-paradigm","display_name":"Programming paradigm","score":0.5368960499763489},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.5077308416366577},{"id":"https://openalex.org/keywords/interoperability","display_name":"Interoperability","score":0.45791569352149963},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.33949756622314453},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.28787028789520264},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.2627943456172943}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8150862455368042},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.7391349077224731},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.6770385503768921},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6432515978813171},{"id":"https://openalex.org/C34165917","wikidata":"https://www.wikidata.org/wiki/Q188267","display_name":"Programming paradigm","level":2,"score":0.5368960499763489},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5077308416366577},{"id":"https://openalex.org/C20136886","wikidata":"https://www.wikidata.org/wiki/Q749647","display_name":"Interoperability","level":2,"score":0.45791569352149963},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.33949756622314453},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.28787028789520264},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.2627943456172943},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/bdcat.2018.00010","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bdcat.2018.00010","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2018 IEEE/ACM 5th International Conference on Big Data Computing Applications and Technologies (BDCAT)","raw_type":"proceedings-article"},{"id":"pmh:oai:e-archivo.uc3m.es:10016/34501","is_oa":true,"landing_page_url":"http://hdl.handle.net/10016/34501","pdf_url":"https://e-archivo.uc3m.es/bitstreams/a288dee3-7cc2-4d69-8a3f-bec4a212bb3e/download","source":{"id":"https://openalex.org/S4306400817","display_name":"e-Archivo (Carlos III University of Madrid)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I50357001","host_organization_name":"Universidad Carlos III de Madrid","host_organization_lineage":["https://openalex.org/I50357001"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"pmh:oai:e-archivo.uc3m.es:10016/34501","is_oa":true,"landing_page_url":"http://hdl.handle.net/10016/34501","pdf_url":"https://e-archivo.uc3m.es/bitstreams/a288dee3-7cc2-4d69-8a3f-bec4a212bb3e/download","source":{"id":"https://openalex.org/S4306400817","display_name":"e-Archivo (Carlos III University of Madrid)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I50357001","host_organization_name":"Universidad Carlos III de Madrid","host_organization_lineage":["https://openalex.org/I50357001"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/conferenceObject"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.5400000214576721,"display_name":"Industry, innovation and infrastructure"}],"awards":[{"id":"https://openalex.org/G1015216249","display_name":null,"funder_award_id":"DE-AC02-06CH1135","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G1116540683","display_name":null,"funder_award_id":"DE-AC02-06CH11","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G1751644051","display_name":null,"funder_award_id":"DE-AC02-06CH11357","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G1844751952","display_name":null,"funder_award_id":"DE-AC02_06CH11357","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G2777053550","display_name":null,"funder_award_id":"AC02-06CH11357","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G3000213308","display_name":null,"funder_award_id":"DE-AC02-06CH113","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G3801222974","display_name":null,"funder_award_id":"DE-AC02-06CH1135","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G4145457401","display_name":null,"funder_award_id":"DE-AC02-06CH113","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G498139845","display_name":null,"funder_award_id":"DE-AC02","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G5302836848","display_name":null,"funder_award_id":"DE-DC000122495","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G5953865680","display_name":null,"funder_award_id":"TIN2016-79637-P","funder_id":"https://openalex.org/F4320326262","funder_display_name":"Ministerio de Asuntos Econ\u00f3micos y Transformaci\u00f3n Digital, Gobierno de Espa\u00f1a"},{"id":"https://openalex.org/G6085113958","display_name":null,"funder_award_id":"E-AC02-06CH11357","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G650626725","display_name":null,"funder_award_id":"E-AC02-06CH11357","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G6558272803","display_name":null,"funder_award_id":"DE-AC02","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G6848031779","display_name":null,"funder_award_id":"06CH11357","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G6918803902","display_name":null,"funder_award_id":"06CH11357","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G7351994996","display_name":null,"funder_award_id":"DE-AC02-06CH11357","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7954425250","display_name":null,"funder_award_id":"DE-AC02-06CH11357","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G8143874970","display_name":null,"funder_award_id":"AC02-06CH11357","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G8314672274","display_name":null,"funder_award_id":"DE-AC02-06CH11","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G8414908677","display_name":null,"funder_award_id":"DE-AC0","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G969889393","display_name":null,"funder_award_id":"DE-AC02-","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306084","display_name":"U.S. Department of Energy","ror":"https://ror.org/01bj3aw27"},{"id":"https://openalex.org/F4320326262","display_name":"Ministerio de Asuntos Econ\u00f3micos y Transformaci\u00f3n Digital, Gobierno de Espa\u00f1a","ror":"https://ror.org/03sv46s19"},{"id":"https://openalex.org/F4320332359","display_name":"Office of Science","ror":"https://ror.org/00mmn6b08"},{"id":"https://openalex.org/F4320337506","display_name":"Advanced Scientific Computing Research","ror":"https://ror.org/0012c7r22"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2908586127.pdf","grobid_xml":"https://content.openalex.org/works/W2908586127.grobid-xml"},"referenced_works_count":37,"referenced_works":["https://openalex.org/W1581677478","https://openalex.org/W1859126261","https://openalex.org/W1868798185","https://openalex.org/W1999329795","https://openalex.org/W2008216781","https://openalex.org/W2009453306","https://openalex.org/W2046449274","https://openalex.org/W2061387982","https://openalex.org/W2063368677","https://openalex.org/W2071704971","https://openalex.org/W2097599414","https://openalex.org/W2131975293","https://openalex.org/W2154702874","https://openalex.org/W2166837162","https://openalex.org/W2341381023","https://openalex.org/W2382024644","https://openalex.org/W2488548223","https://openalex.org/W2550585696","https://openalex.org/W2558413145","https://openalex.org/W2588997145","https://openalex.org/W2599869013","https://openalex.org/W2604553178","https://openalex.org/W2612323846","https://openalex.org/W2613579336","https://openalex.org/W2622770743","https://openalex.org/W2728571383","https://openalex.org/W2731949733","https://openalex.org/W2789811472","https://openalex.org/W2791726704","https://openalex.org/W2796533053","https://openalex.org/W2800893514","https://openalex.org/W2807629360","https://openalex.org/W2963747422","https://openalex.org/W3102741717","https://openalex.org/W6679815717","https://openalex.org/W6722683717","https://openalex.org/W6740652490"],"related_works":["https://openalex.org/W2093262417","https://openalex.org/W4312626803","https://openalex.org/W2123131699","https://openalex.org/W4390608645","https://openalex.org/W650116260","https://openalex.org/W2766461310","https://openalex.org/W4247566972","https://openalex.org/W4388692845","https://openalex.org/W3202731209","https://openalex.org/W3211874991"],"abstract_inverted_index":{"Today's":[0],"scientific":[1,39],"applications":[2],"are":[3,74],"increasingly":[4],"relying":[5],"on":[6,93,192],"a":[7,20,94,120],"variety":[8],"of":[9,109,134,156,176,184],"data":[10,24,60,114,129],"sources,":[11],"storage":[12],"facilities,":[13],"and":[14,17,26,49,56,62,64,86,101,116,131,165,189,197],"computing":[15,40],"infrastructures,":[16],"there":[18],"is":[19,41],"growing":[21],"demand":[22],"for":[23,28,38,51],"analysis":[25],"visualization":[27],"these":[29],"applications.":[30],"In":[31],"this":[32],"context,":[33],"exploiting":[34],"Big":[35,71,96,186],"Data":[36,72,97,187],"frameworks":[37],"an":[42,77,103],"opportunity":[43],"to":[44,105,147,150],"incorporate":[45],"high-level":[46,185],"libraries,":[47],"platforms,":[48],"algorithms":[50],"machine":[52],"learning,":[53],"graph":[54],"processing,":[55],"streaming;":[57],"inherit":[58],"their":[59],"awareness":[61],"fault-tolerance;":[63],"increase":[65],"productivity.":[66],"Nevertheless,":[67],"limitations":[68],"exist":[69],"when":[70],"platforms":[73],"integrated":[75],"with":[76,119],"HPC":[78],"environment,":[79],"namely":[80],"poor":[81],"scalability,":[82],"severe":[83],"memory":[84],"overhead,":[85],"huge":[87],"development":[88],"effort.":[89],"This":[90],"paper":[91],"focuses":[92],"popular":[95],"framework":[98],"-Apache":[99],"Spark-":[100],"proposes":[102],"architecture":[104],"support":[106],"the":[107,128,141,145,151,177,181],"integration":[108],"highly":[110],"scalable":[111],"MPI":[112,152,166],"block-based":[113],"models":[115],"communication":[117],"patterns":[118],"map-reduce-based":[121],"programming":[122,132],"model.":[123],"The":[124,154],"resulting":[125],"platform":[126],"preserves":[127],"abstraction":[130],"interface":[133],"Spark,":[135,193],"without":[136,194],"conducting":[137],"any":[138],"changes":[139],"in":[140],"framework,":[142],"but":[143],"allows":[144],"user":[146],"delegate":[148],"operations":[149],"layer.":[153],"evaluation":[155],"our":[157,161],"prototype":[158],"shows":[159],"that":[160],"approach":[162],"integrates":[163],"Spark":[164],"efficiently":[167],"at":[168],"scale,":[169],"so":[170],"end":[171],"users":[172],"can":[173],"take":[174],"advantage":[175],"productivity":[178],"facilitated":[179],"by":[180],"rich":[182],"ecosystem":[183],"tools":[188],"libraries":[190],"based":[191],"compromising":[195],"efficiency":[196],"scalability.":[198]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":5}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}