From 7665c639bd34392f5d95c177b520d48b9ffa40f4 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Mon, 23 Sep 2024 11:52:04 +0200 Subject: [PATCH] Use Unity Catalog for pipelines in the default-python template (#1766) ## Summary Enables Unity Catalog for pipelines in the default template. Pipelines will default to non-Unity Catalog pipelines if a catalog is not specified. *Small caveat*: there are cases where admins lock down the default catalog of a workspace and don't allow the creation of a new schema there. If that happens, the pipeline would fail at runtime with a clear error indicating what happened. ("PERMISSION_DENIED: User does not have CREATE SCHEMA on Catalog 'main'."). I've seen this with an internal Databricks workspace, where creating new non-UC schemas wasn't locked down, but creation in the `main` was. ## Testing - Validated on a non-UC + UC workspace. The catalog selection logic here is the same as applied for the SQL templates. --- .../resources/{{.project_name}}_pipeline.yml.tmpl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl index 4b8f74d1..bf469046 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl @@ -3,6 +3,12 @@ resources: pipelines: {{.project_name}}_pipeline: name: {{.project_name}}_pipeline + {{- if eq default_catalog ""}} + ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: + # catalog: catalog_name + {{- else}} + catalog: {{default_catalog}} + {{- end}} target: {{.project_name}}_${bundle.environment} libraries: - notebook: