{"id":"https://openalex.org/W4411374362","doi":"https://doi.org/10.1145/3722212.3725084","title":"Blink Twice - Automatic Workload Pinning and Regression Detection for Versionless Apache Spark using Retries","display_name":"Blink Twice - Automatic Workload Pinning and Regression Detection for Versionless Apache Spark using Retries","publication_year":2025,"publication_date":"2025-06-17","ids":{"openalex":"https://openalex.org/W4411374362","doi":"https://doi.org/10.1145/3722212.3725084"},"language":"en","primary_location":{"id":"doi:10.1145/3722212.3725084","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3722212.3725084","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion of the 2025 International Conference on Management of Data","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Justin Breese","orcid":"https://orcid.org/0009-0007-8364-2592"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Justin Breese","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021334686","display_name":"Vijayan Prabhakaran","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vijayan Prabhakaran","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008074234","display_name":"Martin Grund","orcid":"https://orcid.org/0009-0001-1655-0133"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Martin Grund","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031997442","display_name":"Stefania Leone","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Stefania Leone","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5105402419","display_name":"Amit Shukla","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Amit Shukla","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073379366","display_name":"Michael Armbrust","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Michael Armbrust","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000259060","display_name":"Reynold Xin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Reynold Xin","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005554337","display_name":"Matei Zaharia","orcid":"https://orcid.org/0000-0002-7547-7204"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Matei Zaharia","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007052556","display_name":"Lennart C.L. Kats","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lennart Kats","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Sung Chiu","orcid":"https://orcid.org/0009-0000-2712-9814"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sung Chiu","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007943169","display_name":"\u0422.\u041d. \u0420\u043e\u043c\u0430\u043d\u043e\u0432\u0430","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tatiana Romanova","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5118382441","display_name":"Philip Nord","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Philip Nord","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Mitchell Webster","orcid":"https://orcid.org/0009-0005-4398-6265"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mitchell Webster","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Chris Munson","orcid":"https://orcid.org/0009-0003-1902-2699"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chris Munson","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Bo Pang","orcid":"https://orcid.org/0009-0006-0671-9878"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bo Pang","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5109634295","display_name":"David Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"David Ma","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":16,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.19314251,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"103","last_page":"106"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.9959999918937683,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9959999918937683,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.8435478210449219},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.7788412570953369},{"id":"https://openalex.org/keywords/regression","display_name":"Regression","score":0.6639175415039062},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5383196473121643},{"id":"https://openalex.org/keywords/regression-analysis","display_name":"Regression analysis","score":0.4478597640991211},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4404953718185425},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.24172881245613098},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.18220952153205872},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.1794510781764984},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.16919907927513123}],"concepts":[{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.8435478210449219},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.7788412570953369},{"id":"https://openalex.org/C83546350","wikidata":"https://www.wikidata.org/wiki/Q1139051","display_name":"Regression","level":2,"score":0.6639175415039062},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5383196473121643},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.4478597640991211},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4404953718185425},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.24172881245613098},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.18220952153205872},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.1794510781764984},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.16919907927513123},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3722212.3725084","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3722212.3725084","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion of the 2025 International Conference on Management of Data","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":2,"referenced_works":["https://openalex.org/W2038412523","https://openalex.org/W4404181355"],"related_works":["https://openalex.org/W31220157","https://openalex.org/W2312753042","https://openalex.org/W4289356671","https://openalex.org/W2389155397","https://openalex.org/W2165884543","https://openalex.org/W3186837933","https://openalex.org/W2368989808","https://openalex.org/W1969346022","https://openalex.org/W2034959125","https://openalex.org/W2355687852"],"abstract_inverted_index":{"For":[0],"many":[1],"users":[2],"of":[3,50,135,138],"Apache":[4,139],"Spark,":[5,26,112],"managing":[6],"Spark":[7,56,61,65,75,83,96,105,115],"version":[8,124],"upgrades":[9,99],"is":[10,22,28],"a":[11,17,127],"significant":[12],"interruption":[13],"that":[14],"typically":[15],"involves":[16],"time-intensive":[18],"code":[19,35],"migration.":[20],"This":[21],"mainly":[23],"because":[24],"in":[25,103],"there":[27],"no":[29],"clear":[30],"separation":[31],"between":[32],"the":[33,37,70,74,122,136],"application":[34,72],"and":[36,100,119],"engine":[38,76,84],"code,":[39],"making":[40],"it":[41],"hard":[42],"to":[43,67,80],"manage":[44],"them":[45],"independently":[46],"(dependency":[47],"clashes,":[48],"use":[49],"internal":[51],"APIs).":[52],"In":[53,86],"Databricks'":[54],"Serverless":[55],"offering,":[57],"we":[58,63,89],"introduced":[59],"Versionless":[60,111],"where":[62],"leverage":[64],"Connect":[66,97],"fully":[68,128],"decouple":[69],"client":[71],"from":[73],"which":[77],"allows":[78],"us":[79],"seamlessly":[81],"upgrade":[82],"versions.":[85],"this":[87],"paper,":[88],"show":[90],"how":[91],"our":[92],"infrastructure":[93],"built":[94],"around":[95],"automatically":[98],"remediates":[101],"failures":[102],"automated":[104],"workloads":[106,116],"without":[107],"any":[108],"interruption.":[109],"Using":[110],"Databricks":[113],"users'":[114],"run":[117],"indefinitely,":[118],"always":[120],"on":[121,126],"latest":[123],"based":[125],"managed":[129],"experience":[130],"while":[131],"retaining":[132],"nearly":[133],"all":[134],"programmability":[137],"Spark.":[140]},"counts_by_year":[],"updated_date":"2026-04-17T18:11:37.981687","created_date":"2025-10-10T00:00:00"}