{"id":"https://openalex.org/W4411374461","doi":"https://doi.org/10.1145/3722212.3724433","title":"Databricks Lakeguard: Supporting Fine-grained Access Control and Multi-user Capabilities for Apache Spark Workloads","display_name":"Databricks Lakeguard: Supporting Fine-grained Access Control and Multi-user Capabilities for Apache Spark Workloads","publication_year":2025,"publication_date":"2025-06-17","ids":{"openalex":"https://openalex.org/W4411374461","doi":"https://doi.org/10.1145/3722212.3724433"},"language":"en","primary_location":{"id":"doi:10.1145/3722212.3724433","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3722212.3724433","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion of the 2025 International Conference on Management of Data","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008074234","display_name":"Martin Grund","orcid":"https://orcid.org/0009-0001-1655-0133"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Martin Grund","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0001-1655-0133","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031997442","display_name":"Stefania Leone","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Stefania Leone","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0004-1557-3378","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056163057","display_name":"Herman van H\u00f6vell","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Herman van H\u00f6vell","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0003-8807-2526","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5118273620","display_name":"Sven Wagner-Boysen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sven Wagner-Boysen","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0009-1649-386X","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046691787","display_name":"Sebastian Hillig","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sebastian Hillig","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0001-8896-2700","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055465072","display_name":"Hyukjin Kwon","orcid":"https://orcid.org/0000-0002-2277-9524"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hyukjin Kwon","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0000-8876-1506","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027104174","display_name":"David Lewis","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"David Lewis","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0001-8100-5588","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031052528","display_name":"Jakob Mund","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jakob Mund","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0003-6824-1839","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113455196","display_name":"P. Poli","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Polo-Francois Poli","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0007-1075-3064","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049868593","display_name":"Lionel Montrieux","orcid":"https://orcid.org/0000-0002-0524-2581"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lionel Montrieux","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-0524-2581","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5118273618","display_name":"Othon Crelier","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Othon Crelier","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0008-1599-7065","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Xiao Li","orcid":"https://orcid.org/0009-0009-5772-4468"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao Li","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0009-5772-4468","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000259060","display_name":"Reynold Xin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Reynold Xin","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0002-5173-1578","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005554337","display_name":"Matei Zaharia","orcid":"https://orcid.org/0000-0002-7547-7204"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Matei Zaharia","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-7547-7204","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027125531","display_name":"Michalis Petropoulos","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Michalis Petropoulos","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0005-3193-9330","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5118273619","display_name":"Thanos Papathanasiou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Thanos Papathanasiou","raw_affiliation_strings":["Databricks, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0008-5390-3947","affiliations":[{"raw_affiliation_string":"Databricks, San Francisco, CA, USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"418","last_page":"430"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10237","display_name":"Cryptography and Data Security","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11614","display_name":"Cloud Data Security Solutions","score":0.9919000267982483,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.8413538932800293},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7927735447883606},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.6488301753997803},{"id":"https://openalex.org/keywords/access-control","display_name":"Access control","score":0.4482800364494324},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.44427490234375},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.11967560648918152},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.08169415593147278}],"concepts":[{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.8413538932800293},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7927735447883606},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.6488301753997803},{"id":"https://openalex.org/C527821871","wikidata":"https://www.wikidata.org/wiki/Q228502","display_name":"Access control","level":2,"score":0.4482800364494324},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.44427490234375},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.11967560648918152},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.08169415593147278}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3722212.3724433","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3722212.3724433","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion of the 2025 International Conference on Management of Data","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W1993892970","https://openalex.org/W2038412523","https://openalex.org/W2085772207","https://openalex.org/W2087834864","https://openalex.org/W2096745467","https://openalex.org/W3010301034","https://openalex.org/W3085477028","https://openalex.org/W3112202346","https://openalex.org/W4210929589","https://openalex.org/W4312262779","https://openalex.org/W4318148206","https://openalex.org/W4398234408","https://openalex.org/W4404181355","https://openalex.org/W4404181470"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W1975949872","https://openalex.org/W3159871278","https://openalex.org/W2230552005","https://openalex.org/W2905242764","https://openalex.org/W4391094140","https://openalex.org/W3109411864","https://openalex.org/W3017846737"],"abstract_inverted_index":{"Today,":[0],"upgrading":[1],"Apache":[2],"Spark":[3,22,44,56,62,80,124,149,169,176,179,188,267,270],"versions":[4,151],"typically":[5],"involves":[6],"significant":[7,226],"effort,":[8],"with":[9,160,166,214],"unclear":[10],"investment":[11],"requirements,":[12],"including":[13],"trial":[14],"and":[15,32,95,130,135,204,213,241,269,286],"error.":[16],"This":[17,222],"is":[18,24,89],"mainly":[19],"because":[20],"in":[21,39,60,275],"there":[23],"no":[25],"clear":[26],"separation":[27],"between":[28],"the":[29,33,40,61,137,154,172,175,187,195,220,232,258,291],"application":[30],"code":[31,216],"engine":[34,177],"code.":[35],"Apart":[36],"from":[37,105,108,147,174,201,219,245],"changes":[38,46,64,217],"public":[41],"API,":[42],"any":[43],"internal":[45,66],"may":[47,53,73],"affect":[48,74],"user":[49,138,145,292],"workloads":[50,165],"as":[51],"users":[52,81,191],"rely":[54],"on":[55],"internals:":[57],"bug":[58,93],"fixes":[59],"engine,":[63],"to":[65,85,101,117,152,184,194,228,247],"APIs,":[67],"library":[68],"upgrades,":[69],"or":[70],"language":[71],"upgrades":[72,282],"customer":[75],"workloads.":[76],"As":[77],"a":[78,119,167],"result,":[79],"are":[82],"often":[83],"reluctant":[84],"upgrade.":[86],"The":[87],"downside":[88],"that":[90,157],"performance":[91],"improvements,":[92],"fixes,":[94],"new":[96],"features":[97,197],"take":[98],"significantly":[99],"longer":[100],"adopt,":[102],"preventing":[103],"customers":[104],"quickly":[106],"benefiting":[107],"these":[109],"improvements.":[110],"In":[111,250],"addition,":[112],"it":[113],"increases":[114],"engineering":[115,243],"complexity":[116],"manage":[118,280],"large":[120],"number":[121],"of":[122,261],"different":[123],"versions.":[125],"For":[126],"Databricks":[127,183,229],"serverless":[128],"jobs":[129],"notebooks,":[131],"we":[132,162,253,279],"fundamentally":[133],"transformed":[134],"simplified":[136],"experience":[139,293],"when":[140],"using":[141,178],"Spark.":[142],"We":[143],"shifted":[144],"focus":[146],"managing":[148,153],"runtime":[150],"stable":[155],"API":[156],"they":[158],"integrate":[159],"-":[161],"created":[163],"client-versioned":[164],"versionless":[168,262],"server.":[170],"Decoupling":[171],"client":[173],"Connect":[180],"has":[181],"enabled":[182],"automatically":[185],"upgrade":[186],"server,":[189],"providing":[190],"faster":[192],"access":[193],"latest":[196],"while":[198],"minimizing":[199],"disruptions":[200],"both":[202],"intentional":[203],"unintentional":[205],"breaking":[206],"changes,":[207],"all":[208],"without":[209],"compromising":[210],"workload":[211],"compatibility":[212],"zero":[215],"needed":[218],"user.":[221],"approach":[223],"also":[224],"offers":[225],"benefits":[227],"by":[230,273],"streamlining":[231],"release":[233],"process,":[234],"consolidating":[235],"usage":[236],"onto":[237],"fewer":[238],"server":[239],"versions,":[240],"reducing":[242],"overhead":[244],"needing":[246],"backport":[248],"changes.":[249],"this":[251],"demonstration,":[252],"will":[254],"first":[255],"briefly":[256],"introduce":[257],"architectural":[259],"foundation":[260],"Spark,":[263],"leveraging":[264],"Databricks'":[265],"multi-user":[266],"compute":[268],"Connect,":[271],"followed":[272],"describing":[274],"more":[276],"detail":[277],"how":[278],"seamless":[281],"for":[283],"our":[284],"customers,":[285],"finally":[287],"talk":[288],"about":[289],"what":[290],"is.":[294]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}