{"id":"https://openalex.org/W2782923814","doi":"https://doi.org/10.1109/bigdata.2017.8258209","title":"Understanding and improving disk-based intermediate data caching in Spark","display_name":"Understanding and improving disk-based intermediate data caching in Spark","publication_year":2017,"publication_date":"2017-12-01","ids":{"openalex":"https://openalex.org/W2782923814","doi":"https://doi.org/10.1109/bigdata.2017.8258209","mag":"2782923814"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata.2017.8258209","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata.2017.8258209","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2017 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039222039","display_name":"Zhang Kaihui","orcid":null},"institutions":[{"id":"https://openalex.org/I146399215","display_name":"University of Tsukuba","ror":"https://ror.org/02956yf07","country_code":"JP","type":"education","lineage":["https://openalex.org/I146399215"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Kaihui Zhang","raw_affiliation_strings":["University of Tsukuba, Tsukuba, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Tsukuba, Tsukuba, Japan","institution_ids":["https://openalex.org/I146399215"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108062798","display_name":"Yusuke Tanimura","orcid":"https://orcid.org/0000-0002-9140-7169"},"institutions":[{"id":"https://openalex.org/I146399215","display_name":"University of Tsukuba","ror":"https://ror.org/02956yf07","country_code":"JP","type":"education","lineage":["https://openalex.org/I146399215"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Yusuke Tanimura","raw_affiliation_strings":["University of Tsukuba, Tsukuba, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Tsukuba, Tsukuba, Japan","institution_ids":["https://openalex.org/I146399215"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027931271","display_name":"Hidemoto Nakada","orcid":"https://orcid.org/0000-0002-8901-2504"},"institutions":[{"id":"https://openalex.org/I146399215","display_name":"University of Tsukuba","ror":"https://ror.org/02956yf07","country_code":"JP","type":"education","lineage":["https://openalex.org/I146399215"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Hidemoto Nakada","raw_affiliation_strings":["University of Tsukuba, Tsukuba, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Tsukuba, Tsukuba, Japan","institution_ids":["https://openalex.org/I146399215"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103752908","display_name":"Hirotaka Ogawa","orcid":null},"institutions":[{"id":"https://openalex.org/I73613424","display_name":"National Institute of Advanced Industrial Science and Technology","ror":"https://ror.org/01703db54","country_code":"JP","type":"government","lineage":["https://openalex.org/I73613424"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Hirotaka Ogawa","raw_affiliation_strings":["National Institute of Advanced Industrial Science and Technology (AIST), Tsukuba, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Institute of Advanced Industrial Science and Technology (AIST), Tsukuba, Japan","institution_ids":["https://openalex.org/I73613424"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"17","issue":null,"first_page":"2508","last_page":"2517"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8661231398582458},{"id":"https://openalex.org/keywords/serialization","display_name":"Serialization","score":0.8354729413986206},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.7016879320144653},{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.628395140171051},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.49298641085624695},{"id":"https://openalex.org/keywords/page-cache","display_name":"Page cache","score":0.4394071698188782},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.382641464471817},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.3633437156677246},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.35040730237960815},{"id":"https://openalex.org/keywords/cpu-cache","display_name":"CPU cache","score":0.2893426716327667},{"id":"https://openalex.org/keywords/cache-algorithms","display_name":"Cache algorithms","score":0.1570998728275299}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8661231398582458},{"id":"https://openalex.org/C52723943","wikidata":"https://www.wikidata.org/wiki/Q1127410","display_name":"Serialization","level":2,"score":0.8354729413986206},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.7016879320144653},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.628395140171051},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.49298641085624695},{"id":"https://openalex.org/C36340418","wikidata":"https://www.wikidata.org/wiki/Q7124288","display_name":"Page cache","level":5,"score":0.4394071698188782},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.382641464471817},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3633437156677246},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.35040730237960815},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.2893426716327667},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.1570998728275299},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata.2017.8258209","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata.2017.8258209","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2017 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320334764","display_name":"Japan Society for the Promotion of Science","ror":"https://ror.org/00hhkn466"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W410850256","https://openalex.org/W2038157364","https://openalex.org/W2038412523","https://openalex.org/W2112182055","https://openalex.org/W2122465391","https://openalex.org/W2131975293","https://openalex.org/W2190012392","https://openalex.org/W2413953908","https://openalex.org/W2475738333","https://openalex.org/W2488548223","https://openalex.org/W2963288913","https://openalex.org/W6614148910","https://openalex.org/W6679815717","https://openalex.org/W6715689577","https://openalex.org/W6996886134"],"related_works":["https://openalex.org/W4234089424","https://openalex.org/W2539712666","https://openalex.org/W2118932116","https://openalex.org/W4235330364","https://openalex.org/W2099372890","https://openalex.org/W2037579040","https://openalex.org/W2509523906","https://openalex.org/W2406137460","https://openalex.org/W2184371594","https://openalex.org/W2006655698"],"abstract_inverted_index":{"Apache":[0],"Spark":[1,31,82],"is":[2,58,128],"a":[3,24,138,151,177,188],"parallel":[4],"data":[5,20,26,36,79,105,117,214],"processing":[6,53],"framework":[7],"that":[8,125],"executes":[9],"fast":[10],"for":[11,91,209],"iterative":[12],"calculations":[13],"and":[14,83,119,145,181,206],"interactive":[15],"processing,":[16],"by":[17,42],"caching":[18,80,106,115,215],"intermediate":[19,78],"in":[21,77,81,104,110,134,192,213,216],"memory":[22,40,144,153,179,190],"with":[23,195],"lineage-based":[25],"recovery":[27],"from":[28],"faults.":[29],"The":[30,122],"system":[32],"can":[33],"also":[34,84],"manage":[35],"sets":[37],"larger":[38],"than":[39,131],"capacity":[41],"placing":[43],"some":[44],"cache":[45],"or":[46],"all":[47],"of":[48,75,89,101,140,143,172],"them":[49],"on":[50,52],"disks":[51,76,90,212],"nodes.":[54],"However,":[55],"the":[56,87,98,156,185],"disadvantage":[57],"potential":[59,207],"performance":[60,186],"degradation":[61],"due":[62],"to":[63,71,85,96,160],"disk":[64,102,132,146],"I/O":[65,133],"and/or":[66],"serialization.":[67],"This":[68],"study":[69],"aims":[70],"clarify":[72],"efficient/inefficient":[73],"use":[74,103,142],"improve":[86],"usability":[88],"end":[92],"users.":[93],"In":[94],"order":[95],"achieve":[97],"purpose,":[99],"influence":[100],"was":[107,147,158],"firstly":[108],"investigated":[109],"various":[111],"aspects,":[112],"such":[113],"as":[114],"options,":[116],"abstractions":[118],"storage":[120],"devices.":[121],"results":[123],"indicate":[124],"serialization":[126],"cost":[127],"dominant":[129],"rather":[130],"most":[135,169],"cases.":[136],"Secondly,":[137],"method":[139,157],"combined":[141],"further":[148],"evaluated":[149],"under":[150,176,187],"high":[152,178],"pressure.":[154],"Then":[155],"improved":[159],"avoid":[161],"an":[162],"excessive":[163],"re-caching":[164],"problem,":[165],"which":[166],"achieved":[167],"at":[168],"20-30%":[170],"reduction":[171],"total":[173],"execution":[174],"time":[175],"pressure":[180],"did":[182],"not":[183],"degrade":[184],"low":[189],"pressure,":[191],"our":[193],"experiment":[194],"4":[196],"machine":[197],"learning":[198],"benchmarks.":[199],"Finally,":[200],"this":[201],"paper":[202],"summarizes":[203],"important":[204],"factors":[205],"improvements":[208],"efficiently":[210],"using":[211],"Spark.":[217]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}