{"id":"https://openalex.org/W2804013462","doi":"https://doi.org/10.1145/3206333.3206339","title":"Automatic Caching Decision for Scientific Dataflow Execution in Apache Spark","display_name":"Automatic Caching Decision for Scientific Dataflow Execution in Apache Spark","alternate_titles":[],"publication_year":2018,"publication_date":"2018-06-15","ids":{"openalex":"https://openalex.org/W2804013462","doi":"https://doi.org/10.1145/3206333.3206339","mag":"2804013462"},"language":"en","primary_location":{"id":"doi:10.1145/3206333.3206339","doi":"https://doi.org/10.1145/3206333.3206339","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3206333.3206339","pdf_url":null,"source":{"id":"https://openalex.org/S7407119911","display_name":"ACM SIGMOD Workshop on Algorithms and Systems for MapReduce and Beyond (BeyondMR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 5th ACM SIGMOD Workshop on Algorithms and Systems for MapReduce and Beyond","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W2804013462"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083311048","display_name":"Vinicius M. Gottin","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Vinicius M. Gottin","raw_affiliation_strings":["Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005001032","display_name":"Edward Pacheco","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Edward Pacheco","raw_affiliation_strings":["Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022851931","display_name":"Jonas Dias","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Jonas Dias","raw_affiliation_strings":["Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045201515","display_name":"Angelo E. M. Ciarlini","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Angelo E. M. Ciarlini","raw_affiliation_strings":["Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technology Research & Innovation Group, Dell EMC, Rio de Janeiro, Rio de Janeiro, Brazil","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101746246","display_name":"Bruno Costa","orcid":"https://orcid.org/0000-0001-9655-8698","observed_orcids":["https://orcid.org/0000-0001-9655-8698"]},"institutions":[{"id":"https://openalex.org/I122140584","display_name":"Universidade Federal do Rio de Janeiro","ror":"https://ror.org/03490as77","country_code":"BR","type":"education","lineage":["https://openalex.org/I122140584"]}],"countries":["BR"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Bruno Costa","raw_affiliation_strings":["Federal University of Rio de Janeiro (UFRJ), Rio de Janeiro, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federal University of Rio de Janeiro (UFRJ), Rio de Janeiro, Rio de Janeiro, Brazil","institution_ids":["https://openalex.org/I122140584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078285231","display_name":"Wagner Vieira","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I122140584","display_name":"Universidade Federal do Rio de Janeiro","ror":"https://ror.org/03490as77","country_code":"BR","type":"education","lineage":["https://openalex.org/I122140584"]}],"countries":["BR"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Wagner Vieira","raw_affiliation_strings":["Federal University of Rio de Janeiro (UFRJ), Rio de Janeiro, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federal University of Rio de Janeiro (UFRJ), Rio de Janeiro, Rio de Janeiro, Brazil","institution_ids":["https://openalex.org/I122140584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083685643","display_name":"Yania Molina Souto","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210142352","display_name":"Laborat\u00f3rio Nacional de Computa\u00e7\u00e3o Cient\u00edfica","ror":"https://ror.org/0498ekt05","country_code":"BR","type":"government","lineage":["https://openalex.org/I4210142352"]}],"countries":["BR"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Yania M. Souto","raw_affiliation_strings":["National Laboratory for Scientific Computing (LNCC), Petr\u00f3polis, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Laboratory for Scientific Computing (LNCC), Petr\u00f3polis, Rio de Janeiro, Brazil","institution_ids":["https://openalex.org/I4210142352"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021862673","display_name":"Paulo F. Pires","orcid":"https://orcid.org/0000-0001-5064-4793","observed_orcids":["https://orcid.org/0000-0001-5064-4793"]},"institutions":[{"id":"https://openalex.org/I122140584","display_name":"Universidade Federal do Rio de Janeiro","ror":"https://ror.org/03490as77","country_code":"BR","type":"education","lineage":["https://openalex.org/I122140584"]}],"countries":["BR"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Paulo Pires","raw_affiliation_strings":["Federal University of Rio de Janeiro (UFRJ), Rio de Janeiro, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federal University of Rio de Janeiro (UFRJ), Rio de Janeiro, Rio de Janeiro, Brazil","institution_ids":["https://openalex.org/I122140584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040979175","display_name":"Fabio Andre Machado Porto","orcid":"https://orcid.org/0000-0002-4597-4832","observed_orcids":["https://orcid.org/0000-0002-4597-4832"]},"institutions":[{"id":"https://openalex.org/I4210142352","display_name":"Laborat\u00f3rio Nacional de Computa\u00e7\u00e3o Cient\u00edfica","ror":"https://ror.org/0498ekt05","country_code":"BR","type":"government","lineage":["https://openalex.org/I4210142352"]}],"countries":["BR"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Fabio Porto","raw_affiliation_strings":["National Laboratory for Scientific Computing (LNCC), Petr\u00f3polis, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Laboratory for Scientific Computing (LNCC), Petr\u00f3polis, Rio de Janeiro, Brazil","institution_ids":["https://openalex.org/I4210142352"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5022178039","display_name":"Joao Guilherme Rittmeyer","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210142352","display_name":"Laborat\u00f3rio Nacional de Computa\u00e7\u00e3o Cient\u00edfica","ror":"https://ror.org/0498ekt05","country_code":"BR","type":"government","lineage":["https://openalex.org/I4210142352"]}],"countries":["BR"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Jo\u00e3o G. Rittmeyer","raw_affiliation_strings":["National Laboratory for Scientific Computing (LNCC), Petr\u00f3polis, Rio de Janeiro, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Laboratory for Scientific Computing (LNCC), Petr\u00f3polis, Rio de Janeiro, Brazil","institution_ids":["https://openalex.org/I4210142352"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.3523,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.82033231,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":86,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.5493000149726868,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.5493000149726868,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.36059999465942383,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.05000000074505806,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/apache-spark","display_name":"Apache Spark","score":0.8550000190734863},{"id":"https://openalex.org/keywords/lazy-evaluation","display_name":"lazy evaluation","score":0.7720000147819519},{"id":"https://openalex.org/keywords/in-memory-data-processing","display_name":"in-memory data processing","score":0.8069999814033508},{"id":"https://openalex.org/keywords/cost-model","display_name":"cost model","score":0.6480000019073486},{"id":"https://openalex.org/keywords/big-data-analytics","display_name":"big data analytics","score":0.5609999895095825}],"concepts":[{"id":"https://openalex.org/C96324660","wikidata":"https://www.wikidata.org/wiki/Q205446","display_name":"Dataflow","level":2,"score":0.9656014442443848},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.9017069339752197},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.6480075120925903},{"id":"https://openalex.org/C176727019","wikidata":"https://www.wikidata.org/wiki/Q1172415","display_name":"Dataflow architecture","level":3,"score":0.6096220016479492},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.5799016356468201},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5701605081558228},{"id":"https://openalex.org/C2776834041","wikidata":"https://www.wikidata.org/wiki/Q25346349","display_name":"Execution model","level":2,"score":0.4314456880092621},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4153191149234772},{"id":"https://openalex.org/C160191386","wikidata":"https://www.wikidata.org/wiki/Q868299","display_name":"Control flow","level":2,"score":0.4144717752933502},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.21310725808143616}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3206333.3206339","doi":"https://doi.org/10.1145/3206333.3206339","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3206333.3206339","pdf_url":null,"source":{"id":"https://openalex.org/S7407119911","display_name":"ACM SIGMOD Workshop on Algorithms and Systems for MapReduce and Beyond (BeyondMR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 5th ACM SIGMOD Workshop on Algorithms and Systems for MapReduce and Beyond","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W1914234","https://openalex.org/W2303347","https://openalex.org/W1579147384","https://openalex.org/W1582870960","https://openalex.org/W1641025018","https://openalex.org/W1998082182","https://openalex.org/W2017802743","https://openalex.org/W2028351695","https://openalex.org/W2038412523","https://openalex.org/W2058791854","https://openalex.org/W2121810937","https://openalex.org/W2131975293","https://openalex.org/W2154676045","https://openalex.org/W2260553848","https://openalex.org/W2477987941","https://openalex.org/W2530144554","https://openalex.org/W2547386789","https://openalex.org/W2573246392","https://openalex.org/W2596091892","https://openalex.org/W2912516925"],"related_works":["https://openalex.org/W7123359113","https://openalex.org/W2963670827","https://openalex.org/W7095008529","https://openalex.org/W3138614542","https://openalex.org/W3008444097","https://openalex.org/W2897922406","https://openalex.org/W4406495598","https://openalex.org/W3126541598","https://openalex.org/W3034750404","https://openalex.org/W3176238650"],"abstract_inverted_index":{"Demands":[0],"for":[1,197],"large-scale":[2],"data":[3,21],"analysis":[4],"and":[5,12,70,193,222,225],"processing":[6,26],"have":[7],"led":[8],"to":[9,84,99,105,168,180,217],"the":[10,43,71,97,113,122,138,153,169,182,198,201,220],"development":[11],"widespread":[13],"adoption":[14],"of":[15,45,88,91,115,124,135,141,186,200,238],"computing":[16],"frameworks":[17],"that":[18,155,212,226],"leverage":[19],"in-memory":[20],"processing,":[22],"largely":[23],"outperforming":[24],"disk-based":[25],"systems.":[27],"One":[28],"such":[29],"framework":[30],"is":[31,50,150,215],"Apache":[32],"Spark,":[33],"which":[34],"adopts":[35],"a":[36,46,102,116,119,131,145,190,194,234],"lazy-evaluation":[37],"execution":[38,44],"mode.":[39],"In":[40,82,172],"this":[41,173],"model,":[42,224],"transformation":[47,73,107],"dataflow":[48,170,191,208,221],"operation":[49,104],"delayed":[51],"until":[52],"its":[53],"results":[54,62,204],"are":[55,63],"required":[56,78],"by":[57,68,79,152],"an":[58,177,206],"action.":[59,81],"Furthermore,":[60],"transformation's":[61],"not":[64],"kept":[65],"in":[66,118,161,219],"memory":[67],"default,":[69],"same":[72],"must":[74],"be":[75,166],"re-executed":[76],"whenever":[77],"another":[80],"order":[83],"spare":[85],"unnecessary":[86],"re-execution":[87],"entire":[89],"pipelines":[90],"frequently":[92],"referenced":[93],"operations,":[94],"Spark":[95],"enables":[96],"programmer":[98],"explicitly":[100],"define":[101],"cache":[103,117,126,142,187],"persist":[106],"results.":[108],"However,":[109],"many":[110],"factors":[111],"affect":[112],"efficiency":[114],"dataflow,":[120],"including":[121],"existence":[123],"other":[125],"operations.":[127,202],"Thus,":[128],"even":[129],"with":[130],"reasonably":[132],"small":[133],"number":[134],"transformations,":[136],"choosing":[137],"optimal":[139,184,236],"combination":[140,185,237],"operations":[143,188],"poses":[144],"nontrivial":[146],"problem.":[147],"The":[148],"problem":[149],"highlighted":[151],"fact":[154],"intuitive":[156,229],"strategies":[157],"--":[158],"especially":[159],"considered":[160],"isolation":[162],"-":[163],"may":[164],"actually":[165],"harmful":[167],"efficiency.":[171],"work,":[174],"we":[175],"present":[176],"automatic":[178],"procedure":[179],"compute":[181],"substantially":[183,235],"given":[189],"definition":[192],"simple":[195],"model":[196],"cost":[199,223],"Our":[203],"over":[205],"astronomy":[207],"use":[209],"case":[210],"show":[211],"our":[213],"algorithm":[214],"resilient":[216],"changes":[218],"it":[227],"outperforms":[228],"strategies,":[230],"consistently":[231],"deciding":[232],"on":[233],"caches.":[239]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2804013462","counts_by_year":[{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":1}],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2018-06-01T00:00:00"}
