{"id":"https://openalex.org/W2804013462","doi":"https://doi.org/10.1145/3206333.3206339","title":"Automatic Caching Decision for Scientific Dataflow Execution in Apache Spark","display_name":"Automatic Caching Decision for Scientific Dataflow Execution in Apache Spark","publication_year":2018,"publication_date":"2018-05-17","ids":{"openalex":"https://openalex.org/W2804013462","doi":"https://doi.org/10.1145/3206333.3206339","mag":"2804013462"},"language":"en","primary_location":{"id":"doi:10.1145/3206333.3206339","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3206333.3206339","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 5th ACM SIGMOD Workshop on Algorithms and Systems for MapReduce and Beyond","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083311048","display_name":"Vinicius M. Gottin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vinicius M. Gottin","raw_affiliation_strings":["Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005001032","display_name":"Edward Pacheco","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Edward Pacheco","raw_affiliation_strings":["Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022851931","display_name":"Jonas Dias","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jonas Dias","raw_affiliation_strings":["Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045201515","display_name":"Angelo E. M. Ciarlini","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Angelo E. M. Ciarlini","raw_affiliation_strings":["Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101746246","display_name":"Bruno Costa","orcid":"https://orcid.org/0000-0001-9655-8698"},"institutions":[{"id":"https://openalex.org/I122140584","display_name":"Universidade Federal do Rio de Janeiro","ror":"https://ror.org/03490as77","country_code":"BR","type":"education","lineage":["https://openalex.org/I122140584"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Bruno Costa","raw_affiliation_strings":["Federal University of Rio de Janeiro (UFRJ), Rio de Janeiro, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federal University of Rio de Janeiro (UFRJ), Rio de Janeiro, Rio de Janeiro, Brazil","institution_ids":["https://openalex.org/I122140584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078285231","display_name":"Wagner Vieira","orcid":null},"institutions":[{"id":"https://openalex.org/I122140584","display_name":"Universidade Federal do Rio de Janeiro","ror":"https://ror.org/03490as77","country_code":"BR","type":"education","lineage":["https://openalex.org/I122140584"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Wagner Vieira","raw_affiliation_strings":["Federal University of Rio de Janeiro (UFRJ), Rio de Janeiro, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federal University of Rio de Janeiro (UFRJ), Rio de Janeiro, Rio de Janeiro, Brazil","institution_ids":["https://openalex.org/I122140584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083685643","display_name":"Yania Molina Souto","orcid":null},"institutions":[{"id":"https://openalex.org/I4210142352","display_name":"Laborat\u00f3rio Nacional de Computa\u00e7\u00e3o Cient\u00edfica","ror":"https://ror.org/0498ekt05","country_code":"BR","type":"government","lineage":["https://openalex.org/I4210142352"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Yania M. Souto","raw_affiliation_strings":["National Laboratory for Scientific Computing (LNCC), Petr\u00f3polis, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Laboratory for Scientific Computing (LNCC), Petr\u00f3polis, Rio de Janeiro, Brazil","institution_ids":["https://openalex.org/I4210142352"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021862673","display_name":"Paulo F. Pires","orcid":"https://orcid.org/0000-0001-5064-4793"},"institutions":[{"id":"https://openalex.org/I122140584","display_name":"Universidade Federal do Rio de Janeiro","ror":"https://ror.org/03490as77","country_code":"BR","type":"education","lineage":["https://openalex.org/I122140584"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Paulo Pires","raw_affiliation_strings":["Federal University of Rio de Janeiro (UFRJ), Rio de Janeiro, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federal University of Rio de Janeiro (UFRJ), Rio de Janeiro, Rio de Janeiro, Brazil","institution_ids":["https://openalex.org/I122140584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040979175","display_name":"F\u00e1bio Porto","orcid":"https://orcid.org/0000-0002-4597-4832"},"institutions":[{"id":"https://openalex.org/I4210142352","display_name":"Laborat\u00f3rio Nacional de Computa\u00e7\u00e3o Cient\u00edfica","ror":"https://ror.org/0498ekt05","country_code":"BR","type":"government","lineage":["https://openalex.org/I4210142352"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Fabio Porto","raw_affiliation_strings":["National Laboratory for Scientific Computing (LNCC), Petr\u00f3polis, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Laboratory for Scientific Computing (LNCC), Petr\u00f3polis, Rio de Janeiro, Brazil","institution_ids":["https://openalex.org/I4210142352"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5022178039","display_name":"Joao Guilherme Rittmeyer","orcid":null},"institutions":[{"id":"https://openalex.org/I4210142352","display_name":"Laborat\u00f3rio Nacional de Computa\u00e7\u00e3o Cient\u00edfica","ror":"https://ror.org/0498ekt05","country_code":"BR","type":"government","lineage":["https://openalex.org/I4210142352"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Jo\u00e3o G. Rittmeyer","raw_affiliation_strings":["National Laboratory for Scientific Computing (LNCC), Petr\u00f3polis, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Laboratory for Scientific Computing (LNCC), Petr\u00f3polis, Rio de Janeiro, Brazil","institution_ids":["https://openalex.org/I4210142352"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":14,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dataflow","display_name":"Dataflow","score":0.9656014442443848},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.9017069339752197},{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.6480075120925903},{"id":"https://openalex.org/keywords/dataflow-architecture","display_name":"Dataflow architecture","score":0.6096220016479492},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.5799016356468201},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5701605081558228},{"id":"https://openalex.org/keywords/execution-model","display_name":"Execution model","score":0.4314456880092621},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.4153191149234772},{"id":"https://openalex.org/keywords/control-flow","display_name":"Control flow","score":0.4144717752933502},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.21310725808143616}],"concepts":[{"id":"https://openalex.org/C96324660","wikidata":"https://www.wikidata.org/wiki/Q205446","display_name":"Dataflow","level":2,"score":0.9656014442443848},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.9017069339752197},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.6480075120925903},{"id":"https://openalex.org/C176727019","wikidata":"https://www.wikidata.org/wiki/Q1172415","display_name":"Dataflow architecture","level":3,"score":0.6096220016479492},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.5799016356468201},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5701605081558228},{"id":"https://openalex.org/C2776834041","wikidata":"https://www.wikidata.org/wiki/Q25346349","display_name":"Execution model","level":2,"score":0.4314456880092621},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4153191149234772},{"id":"https://openalex.org/C160191386","wikidata":"https://www.wikidata.org/wiki/Q868299","display_name":"Control flow","level":2,"score":0.4144717752933502},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.21310725808143616}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3206333.3206339","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3206333.3206339","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 5th ACM SIGMOD Workshop on Algorithms and Systems for MapReduce and Beyond","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W1914234","https://openalex.org/W2303347","https://openalex.org/W1579147384","https://openalex.org/W1582870960","https://openalex.org/W1641025018","https://openalex.org/W1998082182","https://openalex.org/W2017802743","https://openalex.org/W2028351695","https://openalex.org/W2038412523","https://openalex.org/W2121810937","https://openalex.org/W2131975293","https://openalex.org/W2154676045","https://openalex.org/W2260553848","https://openalex.org/W2477987941","https://openalex.org/W2530144554","https://openalex.org/W2547386789","https://openalex.org/W2571928150","https://openalex.org/W2573246392","https://openalex.org/W2596091892","https://openalex.org/W2912516925"],"related_works":["https://openalex.org/W2783505431","https://openalex.org/W4318948654","https://openalex.org/W1657797058","https://openalex.org/W2765088138","https://openalex.org/W1998949396","https://openalex.org/W2040778456","https://openalex.org/W2890278066","https://openalex.org/W4320930501","https://openalex.org/W2030063121","https://openalex.org/W2125820000"],"abstract_inverted_index":{"Demands":[0],"for":[1,197],"large-scale":[2],"data":[3,21],"analysis":[4],"and":[5,12,70,193,222,225],"processing":[6,26],"have":[7],"led":[8],"to":[9,84,99,105,168,180,217],"the":[10,43,71,97,113,122,138,153,169,182,198,201,220],"development":[11],"widespread":[13],"adoption":[14],"of":[15,45,88,91,115,124,135,141,186,200,238],"computing":[16],"frameworks":[17],"that":[18,155,212,226],"leverage":[19],"in-memory":[20],"processing,":[22],"largely":[23],"outperforming":[24],"disk-based":[25],"systems.":[27],"One":[28],"such":[29],"framework":[30],"is":[31,50,150,215],"Apache":[32],"Spark,":[33],"which":[34],"adopts":[35],"a":[36,46,102,116,119,131,145,190,194,234],"lazy-evaluation":[37],"execution":[38,44],"mode.":[39],"In":[40,82,172],"this":[41,173],"model,":[42,224],"transformation":[47,73,107],"dataflow":[48,170,191,208,221],"operation":[49,104],"delayed":[51],"until":[52],"its":[53],"results":[54,62,204],"are":[55,63],"required":[56,78],"by":[57,68,79,152],"an":[58,177,206],"action.":[59,81],"Furthermore,":[60],"transformation's":[61],"not":[64],"kept":[65],"in":[66,118,161,219],"memory":[67],"default,":[69],"same":[72],"must":[74],"be":[75,166],"re-executed":[76],"whenever":[77],"another":[80],"order":[83],"spare":[85],"unnecessary":[86],"re-execution":[87],"entire":[89],"pipelines":[90],"frequently":[92],"referenced":[93],"operations,":[94],"Spark":[95],"enables":[96],"programmer":[98],"explicitly":[100],"define":[101],"cache":[103,117,126,142,187],"persist":[106],"results.":[108],"However,":[109],"many":[110],"factors":[111],"affect":[112],"efficiency":[114],"dataflow,":[120],"including":[121],"existence":[123],"other":[125],"operations.":[127,202],"Thus,":[128],"even":[129],"with":[130],"reasonably":[132],"small":[133],"number":[134],"transformations,":[136],"choosing":[137],"optimal":[139,184,236],"combination":[140,185,237],"operations":[143,188],"poses":[144],"nontrivial":[146],"problem.":[147],"The":[148],"problem":[149],"highlighted":[151],"fact":[154],"intuitive":[156,229],"strategies":[157],"--":[158],"especially":[159],"considered":[160],"isolation":[162],"-":[163],"may":[164],"actually":[165],"harmful":[167],"efficiency.":[171],"work,":[174],"we":[175],"present":[176],"automatic":[178],"procedure":[179],"compute":[181],"substantially":[183,235],"given":[189],"definition":[192],"simple":[195],"model":[196],"cost":[199,223],"Our":[203],"over":[205],"astronomy":[207],"use":[209],"case":[210],"show":[211],"our":[213],"algorithm":[214],"resilient":[216],"changes":[218],"it":[227],"outperforms":[228],"strategies,":[230],"consistently":[231],"deciding":[232],"on":[233],"caches.":[239]},"counts_by_year":[{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}