From 6ae353d84bf5f2a824f395a0bd56a20af634d3ca Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 5 Mar 2025 15:19:33 +0100 Subject: [PATCH 1/2] Use schema field for pipeline in builtin template (#2347) ## Changes The `schema` field implies the lifecycle of tables is no longer tied to the lifecycle of the pipeline, as was the case with the `target` field. More information about using the "catalog" and "schema" properties can be found here: https://docs.databricks.com/en/delta-live-tables/target-schema.html ## Tests n/a --------- Co-authored-by: Lennart Kats (databricks) --- .../default-python/classic/out.compare-vs-serverless.diff | 2 +- .../my_default_python/resources/my_default_python.pipeline.yml | 2 +- .../default-python/serverless-customcatalog/output.txt | 2 +- .../my_default_python/resources/my_default_python.pipeline.yml | 2 +- integration/bundle/testdata/default_python/bundle_summary.txt | 2 +- .../resources/{{.project_name}}.pipeline.yml.tmpl | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index 6890badf0..8fdcb8359 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -48,7 +48,7 @@ - catalog: main + ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: + # catalog: catalog_name - target: my_default_python_${bundle.target} + schema: my_default_python_${bundle.target} - serverless: true libraries: - notebook: diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml index 4176f765d..d07d4fa6c 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml @@ -5,7 +5,7 @@ resources: name: my_default_python_pipeline ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: # catalog: catalog_name - target: my_default_python_${bundle.target} + schema: my_default_python_${bundle.target} libraries: - notebook: path: ../src/dlt_pipeline.ipynb diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt index 30726013b..a6a92dfd4 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt @@ -18,5 +18,5 @@ See also the documentation at https://docs.databricks.com/dev-tools/bundles/inde - ## Catalog is required for serverless compute - catalog: main + catalog: customcatalog - target: my_default_python_${bundle.target} + schema: my_default_python_${bundle.target} serverless: true diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml index 6dac62ded..f8a0521db 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml @@ -5,7 +5,7 @@ resources: name: my_default_python_pipeline ## Catalog is required for serverless compute catalog: main - target: my_default_python_${bundle.target} + schema: my_default_python_${bundle.target} serverless: true libraries: - notebook: diff --git a/integration/bundle/testdata/default_python/bundle_summary.txt b/integration/bundle/testdata/default_python/bundle_summary.txt index 968009759..0b4cbb181 100644 --- a/integration/bundle/testdata/default_python/bundle_summary.txt +++ b/integration/bundle/testdata/default_python/bundle_summary.txt @@ -164,7 +164,7 @@ } ], "name": "[dev [USERNAME]] project_name_$UNIQUE_PRJ_pipeline", - "target": "project_name_$UNIQUE_PRJ_dev", + "schema": "project_name_$UNIQUE_PRJ_dev", "url": "[DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID]" } } diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl index 024c1ab15..2e858d681 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl @@ -13,7 +13,7 @@ resources: {{- else}} catalog: {{default_catalog}} {{- end}} - target: {{.project_name}}_${bundle.target} + schema: {{.project_name}}_${bundle.target} {{- if $with_serverless }} serverless: true {{- end}} From a24a7f573844573dd9ea6fbca4cb79cb8da1fa58 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 5 Mar 2025 19:50:59 +0530 Subject: [PATCH 2/2] Remove omitempty tag for exit code and execution time in telemetry (#2433) --- libs/telemetry/protos/databricks_cli_log.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/telemetry/protos/databricks_cli_log.go b/libs/telemetry/protos/databricks_cli_log.go index 9e4e59596..64baa6b38 100644 --- a/libs/telemetry/protos/databricks_cli_log.go +++ b/libs/telemetry/protos/databricks_cli_log.go @@ -24,10 +24,12 @@ type ExecutionContext struct { FromWebTerminal bool `json:"from_web_terminal,omitempty"` // Time taken for the CLI command to execute. - ExecutionTimeMs int64 `json:"execution_time_ms,omitempty"` + // We want to serialize the zero value as well so the omitempty tag is not set. + ExecutionTimeMs int64 `json:"execution_time_ms"` // Exit code of the CLI command. - ExitCode int64 `json:"exit_code,omitempty"` + // We want to serialize the zero value as well so the omitempty tag is not set. + ExitCode int64 `json:"exit_code"` } type CliTestEvent struct {