From 6ae353d84bf5f2a824f395a0bd56a20af634d3ca Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 5 Mar 2025 15:19:33 +0100 Subject: [PATCH] Use schema field for pipeline in builtin template (#2347) ## Changes The `schema` field implies the lifecycle of tables is no longer tied to the lifecycle of the pipeline, as was the case with the `target` field. More information about using the "catalog" and "schema" properties can be found here: https://docs.databricks.com/en/delta-live-tables/target-schema.html ## Tests n/a --------- Co-authored-by: Lennart Kats (databricks) --- .../default-python/classic/out.compare-vs-serverless.diff | 2 +- .../my_default_python/resources/my_default_python.pipeline.yml | 2 +- .../default-python/serverless-customcatalog/output.txt | 2 +- .../my_default_python/resources/my_default_python.pipeline.yml | 2 +- integration/bundle/testdata/default_python/bundle_summary.txt | 2 +- .../resources/{{.project_name}}.pipeline.yml.tmpl | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index 6890badf0..8fdcb8359 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -48,7 +48,7 @@ - catalog: main + ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: + # catalog: catalog_name - target: my_default_python_${bundle.target} + schema: my_default_python_${bundle.target} - serverless: true libraries: - notebook: diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml index 4176f765d..d07d4fa6c 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml @@ -5,7 +5,7 @@ resources: name: my_default_python_pipeline ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: # catalog: catalog_name - target: my_default_python_${bundle.target} + schema: my_default_python_${bundle.target} libraries: - notebook: path: ../src/dlt_pipeline.ipynb diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt index 30726013b..a6a92dfd4 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt @@ -18,5 +18,5 @@ See also the documentation at https://docs.databricks.com/dev-tools/bundles/inde - ## Catalog is required for serverless compute - catalog: main + catalog: customcatalog - target: my_default_python_${bundle.target} + schema: my_default_python_${bundle.target} serverless: true diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml index 6dac62ded..f8a0521db 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml @@ -5,7 +5,7 @@ resources: name: my_default_python_pipeline ## Catalog is required for serverless compute catalog: main - target: my_default_python_${bundle.target} + schema: my_default_python_${bundle.target} serverless: true libraries: - notebook: diff --git a/integration/bundle/testdata/default_python/bundle_summary.txt b/integration/bundle/testdata/default_python/bundle_summary.txt index 968009759..0b4cbb181 100644 --- a/integration/bundle/testdata/default_python/bundle_summary.txt +++ b/integration/bundle/testdata/default_python/bundle_summary.txt @@ -164,7 +164,7 @@ } ], "name": "[dev [USERNAME]] project_name_$UNIQUE_PRJ_pipeline", - "target": "project_name_$UNIQUE_PRJ_dev", + "schema": "project_name_$UNIQUE_PRJ_dev", "url": "[DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID]" } } diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl index 024c1ab15..2e858d681 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl @@ -13,7 +13,7 @@ resources: {{- else}} catalog: {{default_catalog}} {{- end}} - target: {{.project_name}}_${bundle.target} + schema: {{.project_name}}_${bundle.target} {{- if $with_serverless }} serverless: true {{- end}}