{"id":"https://openalex.org/W4392490377","doi":"https://doi.org/10.14778/3681954.3682021","title":"A Spark Optimizer for Adaptive, Fine-Grained Parameter Tuning","display_name":"A Spark Optimizer for Adaptive, Fine-Grained Parameter Tuning","publication_year":2024,"publication_date":"2024-07-01","ids":{"openalex":"https://openalex.org/W4392490377","doi":"https://doi.org/10.14778/3681954.3682021"},"language":"en","primary_location":{"id":"doi:10.14778/3681954.3682021","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3681954.3682021","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2403.00995","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057626684","display_name":"Chenghao Lyu","orcid":null},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Chenghao Lyu","raw_affiliation_strings":["University of Massachusetts, Amherst"],"affiliations":[{"raw_affiliation_string":"University of Massachusetts, Amherst","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021386906","display_name":"Qi Fan","orcid":"https://orcid.org/0000-0002-4056-3997"},"institutions":[{"id":"https://openalex.org/I142476485","display_name":"\u00c9cole Polytechnique","ror":"https://ror.org/05hy3tk52","country_code":"FR","type":"education","lineage":["https://openalex.org/I142476485","https://openalex.org/I4210145102"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Qi Fan","raw_affiliation_strings":["Ecole Polytechnique"],"affiliations":[{"raw_affiliation_string":"Ecole Polytechnique","institution_ids":["https://openalex.org/I142476485"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075927615","display_name":"Philippe Guyard","orcid":null},"institutions":[{"id":"https://openalex.org/I142476485","display_name":"\u00c9cole Polytechnique","ror":"https://ror.org/05hy3tk52","country_code":"FR","type":"education","lineage":["https://openalex.org/I142476485","https://openalex.org/I4210145102"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Philippe Guyard","raw_affiliation_strings":["Ecole Polytechnique"],"affiliations":[{"raw_affiliation_string":"Ecole Polytechnique","institution_ids":["https://openalex.org/I142476485"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5109548832","display_name":"Yanlei Diao","orcid":"https://orcid.org/0009-0000-7793-9671"},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yanlei Diao","raw_affiliation_strings":["Ecole Polytechnique and University of Massachusetts, Amherst"],"affiliations":[{"raw_affiliation_string":"Ecole Polytechnique and University of Massachusetts, Amherst","institution_ids":["https://openalex.org/I24603500"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5057626684"],"corresponding_institution_ids":["https://openalex.org/I24603500"],"apc_list":null,"apc_paid":null,"fwci":1.4606,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.76726787,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":"17","issue":"11","first_page":"3565","last_page":"3579"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11245","display_name":"Advanced Numerical Analysis Techniques","score":0.9373999834060669,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11245","display_name":"Advanced Numerical Analysis Techniques","score":0.9373999834060669,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.7844669818878174},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4899381995201111}],"concepts":[{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.7844669818878174},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4899381995201111},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.14778/3681954.3682021","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3681954.3682021","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2403.00995","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.00995","pdf_url":"https://arxiv.org/pdf/2403.00995","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:HAL:hal-04906931v1","is_oa":true,"landing_page_url":"https://inria.hal.science/hal-04906931","pdf_url":"https://inria.hal.science/hal-04906931/document","source":{"id":"https://openalex.org/S4406922461","display_name":"SPIRE - Sciences Po Institutional REpository","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"VLDB 2024 - 50th International Conference on Very Large Databases, Aug 2024, Guangzhou, China. ⟨10.14778/3681954.3682021⟩","raw_type":"Conference papers"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2403.00995","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.00995","pdf_url":"https://arxiv.org/pdf/2403.00995","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1426318481","display_name":null,"funder_award_id":"grant","funder_id":"https://openalex.org/F4320322725","funder_display_name":"China Scholarship Council"},{"id":"https://openalex.org/G2300736770","display_name":null,"funder_award_id":"(CSC)","funder_id":"https://openalex.org/F4320322725","funder_display_name":"China Scholarship Council"},{"id":"https://openalex.org/G8589651859","display_name":null,"funder_award_id":"China","funder_id":"https://openalex.org/F4320322725","funder_display_name":"China Scholarship Council"}],"funders":[{"id":"https://openalex.org/F4320322725","display_name":"China Scholarship Council","ror":"https://ror.org/04atp4p48"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4392490377.pdf","grobid_xml":"https://content.openalex.org/works/W4392490377.grobid-xml"},"referenced_works_count":57,"referenced_works":["https://openalex.org/W1789812421","https://openalex.org/W1979079016","https://openalex.org/W1997188340","https://openalex.org/W2035176107","https://openalex.org/W2038412523","https://openalex.org/W2039652440","https://openalex.org/W2049864887","https://openalex.org/W2053688343","https://openalex.org/W2072620509","https://openalex.org/W2082171780","https://openalex.org/W2090492562","https://openalex.org/W2105947650","https://openalex.org/W2110086534","https://openalex.org/W2119558533","https://openalex.org/W2139072600","https://openalex.org/W2142031898","https://openalex.org/W2151251992","https://openalex.org/W2188062232","https://openalex.org/W2258038925","https://openalex.org/W2289698129","https://openalex.org/W2521550930","https://openalex.org/W2554427106","https://openalex.org/W2613046943","https://openalex.org/W2613206411","https://openalex.org/W2760770811","https://openalex.org/W2807637204","https://openalex.org/W2888254789","https://openalex.org/W2948513753","https://openalex.org/W2970851599","https://openalex.org/W2971285141","https://openalex.org/W3007086929","https://openalex.org/W3030994385","https://openalex.org/W3044147426","https://openalex.org/W3126168625","https://openalex.org/W3127018844","https://openalex.org/W3133029391","https://openalex.org/W3167346145","https://openalex.org/W3173588310","https://openalex.org/W3173622057","https://openalex.org/W3174969457","https://openalex.org/W3176434378","https://openalex.org/W3197977787","https://openalex.org/W3198024709","https://openalex.org/W3198307109","https://openalex.org/W3208735199","https://openalex.org/W3208930906","https://openalex.org/W4206064074","https://openalex.org/W4221062824","https://openalex.org/W4226290930","https://openalex.org/W4249517230","https://openalex.org/W4284700449","https://openalex.org/W4285338532","https://openalex.org/W4289533904","https://openalex.org/W4289706945","https://openalex.org/W4312847577","https://openalex.org/W4381329367","https://openalex.org/W4386528681"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W1975949872","https://openalex.org/W3159871278","https://openalex.org/W2230552005","https://openalex.org/W2905242764","https://openalex.org/W3109411864","https://openalex.org/W3017846737","https://openalex.org/W2390279801","https://openalex.org/W4379407450"],"abstract_inverted_index":{"As":[0],"Spark":[1,24,47,66,126],"becomes":[2],"a":[3,65,115,131],"common":[4],"big":[5],"data":[6],"analytics":[7],"platform,":[8],"its":[9,83],"growing":[10],"complexity":[11],"makes":[12],"automatic":[13],"tuning":[14,26,93,122,141],"of":[15,64,73,123,133,153,170,194,226],"numerous":[16],"parameters":[17,72],"critical":[18],"for":[19,55,156,184],"performance.":[20],"Our":[21],"work":[22],"on":[23,40],"parameter":[25],"is":[27],"particularly":[28],"motivated":[29],"by":[30],"two":[31],"recent":[32],"trends:":[33],"Spark's":[34],"Adaptive":[35],"Query":[36],"Execution":[37],"(AQE)":[38],"based":[39],"runtime":[41],"statistics,":[42],"and":[43,135,163,181,186,219],"the":[44,56,77,88,92,96,140,144,149,167,198,224],"increasingly":[45],"popular":[46],"cloud":[48,157],"deployments":[49],"that":[50,68,203],"make":[51],"cost-performance":[52,108],"reasoning":[53],"crucial":[54],"end":[57],"user.":[58],"This":[59],"paper":[60],"presents":[61],"our":[62,171,221],"design":[63],"optimizer":[67],"controls":[69],"all":[70],"tunable":[71],"each":[74],"query":[75],"in":[76,95,143],"new":[78],"AQE":[79],"architecture":[80],"to":[81,103,106,120,138,232],"explore":[82],"performance":[84,169],"benefits":[85],"and,":[86],"at":[87],"same":[89],"time,":[90],"casts":[91],"problem":[94,142],"theoretically":[97],"sound":[98],"multi-objective":[99],"optimization":[100,136],"(MOO)":[101],"setting":[102,146],"better":[104],"adapt":[105],"user":[107],"preferences.":[109,234],"To":[110],"this":[111],"end,":[112],"we":[113],"propose":[114],"novel":[116],"hybrid":[117],"compile-time/runtime":[118],"approach":[119,222],"multi-granularity":[121],"diverse,":[124],"correlated":[125],"parameters,":[127],"as":[128,130],"well":[129],"suite":[132],"modeling":[134],"techniques":[137],"solve":[139],"MOO":[145,201],"while":[147],"meeting":[148],"stringent":[150],"time":[151,193],"constraint":[152],"1--2":[154],"seconds":[155],"use.":[158],"Evaluation":[159],"results":[160],"using":[161],"TPC-H":[162,185],"TPC-DS":[164],"benchmarks":[165],"demonstrate":[166],"superior":[168,230],"approach:":[172],"(i":[173],")":[174],"When":[175,214],"prioritizing":[176],"latency,":[177],"it":[178],"achieves":[179],"63%":[180],"65%":[182],"reduction":[183],"TPC-DS,":[187],"respectively,":[188],"under":[189],"an":[190],"average":[191],"solving":[192,211],"0.7--0.8":[195],"sec,":[196],"outperforming":[197],"most":[199],"competitive":[200],"method":[202],"reduces":[204],"only":[205],"18--25%":[206],"latency":[207,218],"with":[208],"2.6--15":[209],"sec":[210],"time.":[212],"(ii)":[213],"shifting":[215],"preferences":[216],"between":[217],"cost,":[220],"dominates":[223],"solutions":[225],"alternative":[227],"methods,":[228],"exhibiting":[229],"adaptability":[231],"varying":[233]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}