{"id":"https://openalex.org/W7168288051","doi":"https://doi.org/10.48550/arxiv.2607.11415","title":"Stage-Level Executor Allocation in Apache Spark with Cost-Performance Trade-offs","display_name":"Stage-Level Executor Allocation in Apache Spark with Cost-Performance Trade-offs","publication_year":2026,"publication_date":"2026-07-13","ids":{"openalex":"https://openalex.org/W7168288051","doi":"https://doi.org/10.48550/arxiv.2607.11415"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.11415","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.11415","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.11415","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140601782","display_name":"Miriam Rateike","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rateike, Miriam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140608971","display_name":"Isaac Waweru Wambugu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wambugu, Isaac Waweru","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140691184","display_name":"Celia Cintas","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cintas, Celia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140627939","display_name":"Michael Kaufmann","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kaufmann, Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140698669","display_name":"Ioana Giurgiu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Giurgiu, Ioana","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5140694080","display_name":"Skyler Speakman","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Speakman, Skyler","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.6502000093460083,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.6502000093460083,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.27730000019073486,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10772","display_name":"Distributed systems and fault tolerance","score":0.014600000344216824,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/executor","display_name":"Executor","score":0.8830000162124634},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.8391000032424927},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5184000134468079},{"id":"https://openalex.org/keywords/resource","display_name":"Resource (disambiguation)","score":0.4666000008583069},{"id":"https://openalex.org/keywords/resource-allocation","display_name":"Resource allocation","score":0.4271000027656555},{"id":"https://openalex.org/keywords/tuple","display_name":"Tuple","score":0.3952000141143799}],"concepts":[{"id":"https://openalex.org/C180591056","wikidata":"https://www.wikidata.org/wiki/Q654437","display_name":"Executor","level":2,"score":0.8830000162124634},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.8391000032424927},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7764999866485596},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5184000134468079},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.4666000008583069},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4499000012874603},{"id":"https://openalex.org/C29202148","wikidata":"https://www.wikidata.org/wiki/Q287260","display_name":"Resource allocation","level":2,"score":0.4271000027656555},{"id":"https://openalex.org/C118930307","wikidata":"https://www.wikidata.org/wiki/Q600590","display_name":"Tuple","level":2,"score":0.3952000141143799},{"id":"https://openalex.org/C190839683","wikidata":"https://www.wikidata.org/wiki/Q2448197","display_name":"Train","level":2,"score":0.3849000036716461},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.3774000108242035},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.31029999256134033},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2994999885559082},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.2964000105857849},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.2777000069618225},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.2685000002384186},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.257099986076355},{"id":"https://openalex.org/C2780609101","wikidata":"https://www.wikidata.org/wiki/Q17156588","display_name":"Resource management (computing)","level":2,"score":0.25200000405311584}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.11415","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.11415","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.11415","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.11415","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Allocating":[0],"executors":[1,24,91],"(i.e.":[2],"compute":[3],"resources)":[4],"to":[5,39,79,102,113,134],"distributed":[6],"processing":[7],"systems":[8],"must":[9],"balance":[10],"resource":[11],"costs":[12,32,119],"of":[13,44,47,90,120,126],"scaling-out":[14],"unnecessarily":[15],"against":[16,153],"artificial,":[17],"performance-limiting":[18],"bottlenecks.":[19],"Naive":[20],"approaches":[21],"may":[22,56],"allocate":[23],"at":[25,188],"the":[26,45,53,88,115,158],"application":[27,65],"level,":[28],"which":[29],"have":[30,58],"predictable":[31],"and":[33,118,147,150,162,181],"performance":[34],"but":[35],"are":[36,131],"almost":[37],"guaranteed":[38],"be":[40],"sub-optimal":[41],"for":[42,86,137],"each":[43,138],"thousands":[46],"diverse,":[48],"individual":[49],"stages":[50],"executed":[51],"by":[52],"application.":[54],"Users":[55],"also":[57],"explicit":[59],"preferences,":[60],"such":[61],"as":[62,123],"completing":[63],"an":[64],"within":[66],"a":[67,83,95,121,124,177,189],"specific":[68],"time":[69],"budget":[70],"while":[71],"minimizing":[72],"cost,":[73],"that":[74],"existing":[75],"solutions":[76],"usually":[77],"fail":[78],"support.":[80],"We":[81,141],"propose":[82],"novel":[84],"method":[85],"determining":[87],"number":[89],"per":[92],"stage":[93,122,139],"in":[94],"serverless":[96],"Apache":[97],"Spark":[98],"environment,":[99],"enabling":[100],"users":[101],"specify":[103],"their":[104],"desired":[105],"cost-performance":[106],"tradeoff.":[107],"Our":[108],"approach":[109,144,165],"trains":[110],"tree-ensemble":[111],"models":[112],"estimate":[114],"run":[116],"times":[117],"function":[125],"allocated":[127],"resources.":[128],"These":[129],"estimates":[130],"then":[132],"used":[133],"recommend":[135],"resources":[136],"individually.":[140],"evaluate":[142],"our":[143,164],"on":[145,157,184],"TPC-DS":[146,173],"SQLStorm":[148,186],"benchmarks":[149],"compare":[151],"it":[152],"two":[154],"baselines.":[155],"Depending":[156],"user-defined":[159],"trade-off":[160],"parameter":[161],"setup,":[163],"achieves":[166],"approx.":[167,178,182,190],"50%":[168],"cost":[169],"savings":[170],"across":[171],"103":[172],"queries":[174,187],"with":[175],"only":[176],"16%":[179],"slowdown,":[180],"40.5%":[183],"96":[185],"29%":[191],"slowdown.":[192]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-15T00:00:00"}