{"id":"https://openalex.org/W4363671823","doi":"https://doi.org/10.1145/3588921","title":"Runtime Variation in Big Data Analytics","display_name":"Runtime Variation in Big Data Analytics","publication_year":2023,"publication_date":"2023-05-26","ids":{"openalex":"https://openalex.org/W4363671823","doi":"https://doi.org/10.1145/3588921"},"language":"en","primary_location":{"id":"doi:10.1145/3588921","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3588921","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3588921","source":{"id":"https://openalex.org/S4387289859","display_name":"Proceedings of the ACM on Management of Data","issn_l":"2836-6573","issn":["2836-6573"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Management of Data","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3588921","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101749826","display_name":"Yiwen Zhu","orcid":"https://orcid.org/0009-0005-6857-7505"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yiwen Zhu","raw_affiliation_strings":["Microsoft, Redmond, USA"],"raw_orcid":"https://orcid.org/0009-0005-6857-7505","affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061137265","display_name":"Rathijit Sen","orcid":"https://orcid.org/0000-0003-4736-2837"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rathijit Sen","raw_affiliation_strings":["Microsoft, Redmond, USA"],"raw_orcid":"https://orcid.org/0000-0003-4736-2837","affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061532977","display_name":"Robert F. Horton","orcid":"https://orcid.org/0000-0001-7305-354X"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Robert Horton","raw_affiliation_strings":["Microsoft, Mountain View, USA"],"raw_orcid":"https://orcid.org/0000-0001-7305-354X","affiliations":[{"raw_affiliation_string":"Microsoft, Mountain View, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5017404665","display_name":"John Mark Agosta","orcid":"https://orcid.org/0000-0001-9826-6509"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John Mark Agosta","raw_affiliation_strings":["Microsoft, Mountain View, USA"],"raw_orcid":"https://orcid.org/0000-0001-9826-6509","affiliations":[{"raw_affiliation_string":"Microsoft, Mountain View, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":2.0551,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":{"value":0.88907468,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":97},"biblio":{"volume":"1","issue":"1","first_page":"1","last_page":"20"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12292","display_name":"Graph Theory and Algorithms","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.6576427817344666},{"id":"https://openalex.org/keywords/variation","display_name":"Variation (astronomy)","score":0.605820894241333},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5760159492492676},{"id":"https://openalex.org/keywords/analytics","display_name":"Analytics","score":0.5402691960334778},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.47143426537513733},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.19673225283622742},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.06299841403961182}],"concepts":[{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.6576427817344666},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.605820894241333},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5760159492492676},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.5402691960334778},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.47143426537513733},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.19673225283622742},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.06299841403961182},{"id":"https://openalex.org/C44870925","wikidata":"https://www.wikidata.org/wiki/Q37547","display_name":"Astrophysics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3588921","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3588921","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3588921","source":{"id":"https://openalex.org/S4387289859","display_name":"Proceedings of the ACM on Management of Data","issn_l":"2836-6573","issn":["2836-6573"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Management of Data","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2304.03424","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.03424","pdf_url":"https://arxiv.org/pdf/2304.03424","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3588921","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3588921","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3588921","source":{"id":"https://openalex.org/S4387289859","display_name":"Proceedings of the ACM on Management of Data","issn_l":"2836-6573","issn":["2836-6573"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Management of Data","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.5699999928474426,"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4363671823.pdf","grobid_xml":"https://content.openalex.org/works/W4363671823.grobid-xml"},"referenced_works_count":64,"referenced_works":["https://openalex.org/W95608104","https://openalex.org/W1537220347","https://openalex.org/W1932332931","https://openalex.org/W1984116073","https://openalex.org/W1993433750","https://openalex.org/W2009086942","https://openalex.org/W2019781618","https://openalex.org/W2025549137","https://openalex.org/W2035055162","https://openalex.org/W2037826478","https://openalex.org/W2047280416","https://openalex.org/W2053341340","https://openalex.org/W2060419519","https://openalex.org/W2075914362","https://openalex.org/W2087986722","https://openalex.org/W2090066046","https://openalex.org/W2091765165","https://openalex.org/W2101736851","https://openalex.org/W2105947650","https://openalex.org/W2110086534","https://openalex.org/W2119400430","https://openalex.org/W2127882368","https://openalex.org/W2139250332","https://openalex.org/W2142838865","https://openalex.org/W2150606131","https://openalex.org/W2166859998","https://openalex.org/W2169188831","https://openalex.org/W2189465200","https://openalex.org/W2204432465","https://openalex.org/W2397931483","https://openalex.org/W2404352102","https://openalex.org/W2521550930","https://openalex.org/W2538829354","https://openalex.org/W2604258491","https://openalex.org/W2612261081","https://openalex.org/W2612779005","https://openalex.org/W2613206411","https://openalex.org/W2618188015","https://openalex.org/W2734941459","https://openalex.org/W2802117660","https://openalex.org/W2811487842","https://openalex.org/W2892375914","https://openalex.org/W2948513753","https://openalex.org/W2965018236","https://openalex.org/W2969671095","https://openalex.org/W2985275869","https://openalex.org/W3031504748","https://openalex.org/W3033164579","https://openalex.org/W3033414789","https://openalex.org/W3047439613","https://openalex.org/W3085960807","https://openalex.org/W3105923964","https://openalex.org/W3175742134","https://openalex.org/W3176975793","https://openalex.org/W3188064594","https://openalex.org/W3196442060","https://openalex.org/W3197025661","https://openalex.org/W3204950481","https://openalex.org/W3206998513","https://openalex.org/W4221135215","https://openalex.org/W4243552960","https://openalex.org/W4248173958","https://openalex.org/W6675401881","https://openalex.org/W6687322159"],"related_works":["https://openalex.org/W4322629366","https://openalex.org/W2808989540","https://openalex.org/W2397053934","https://openalex.org/W1039292361","https://openalex.org/W2551093110","https://openalex.org/W2148016376","https://openalex.org/W4237919137","https://openalex.org/W3184179822","https://openalex.org/W3095362084","https://openalex.org/W3003361536"],"abstract_inverted_index":{"The":[0],"dynamic":[1],"nature":[2],"of":[3,31,76,78,116,156,167,193],"resource":[4,143,194],"allocation":[5],"and":[6,34,39,56,64,128,141,148,150,197,207],"runtime":[7,19,74,100,168,205],"conditions":[8],"on":[9,82,164,190,202],"Cloud":[10],"can":[11,182],"result":[12],"in":[13,16],"high":[14],"variability":[15],"a":[17,25,109,203],"job's":[18,204],"across":[20],"multiple":[21],"iterations,":[22],"leading":[23],"to":[24,37,45,49,97,185],"poor":[26],"experience.":[27],"Identifying":[28],"the":[29,73,154,161,191],"sources":[30],"such":[32,136],"variation":[33,75],"being":[35],"able":[36],"predict":[38,98],"adjust":[40,59],"for":[41,170],"them":[42],"is":[43,160],"crucial":[44],"cloud":[46],"service":[47],"providers":[48],"design":[50],"reliable":[51],"data":[52],"processing":[53],"pipelines,":[54],"provision":[55],"allocate":[57],"resources,":[58],"pricing":[60],"services,":[61],"meet":[62],"SLOs":[63],"debug":[65],"performance":[66],"hazards.":[67],"In":[68],"this":[69,159],"paper,":[70],"we":[71,177],"analyze":[72,186],"millions":[77],"production":[79],"Scope":[80],"jobs":[81],"Cosmos,":[83],"an":[84,93,113,119],"exabyte-scale":[85],"internal":[86],"analytics":[87,172],"platform":[88],"at":[89,174],"Microsoft.":[90],"We":[91,133],"propose":[92],"innovative":[94,120],"2-step":[95],"approach":[96],"job":[99,138],"distribution":[101,105],"by":[102],"characterizing":[103],"typical":[104],"shapes":[106],"combined":[107],"with":[108,112],"classification":[110],"model":[111],"average":[114],"accuracy":[115],">96%,":[117],"using":[118],"interpretable":[121],"machine-learning":[122],"algorithm":[123],"out-performing":[124],"traditional":[125],"regression":[126],"models":[127],"better":[129],"capturing":[130],"long":[131],"tails.":[132],"examine":[134,178],"factors":[135],"as":[137],"plan":[139],"characteristics":[140],"inputs,":[142],"allocation,":[144,195],"physical":[145,198],"cluster":[146,199],"heterogeneity":[147],"utilization,":[149],"scheduling":[151],"policies.":[152],"To":[153],"best":[155],"our":[157,180],"knowledge,":[158],"first":[162],"study":[163],"predicting":[165],"categories":[166],"distributions":[169],"enterprise":[171],"workloads":[173],"scale.":[175],"Furthermore,":[176],"how":[179],"methods":[181],"be":[183],"used":[184],"what-if":[187],"scenarios,":[188],"focusing":[189],"impact":[192],"scheduling,":[196],"provisioning":[200],"decisions":[201],"consistency":[206],"predictability.":[208]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3}],"updated_date":"2026-06-14T06:11:07.267592","created_date":"2025-10-10T00:00:00"}